internal/pkg: optional shallow external cure

This saves time and is often more practical even for package maintainers.

Signed-off-by: Ophestra <cat@gensokyo.uk>

internal/pkg/ir.go

@@ -90,7 +90,7 @@ type IContext struct {
 	// returns and must not be exposed directly.
 	w io.Writer
 	// Optional [Artifact] to cureRes cache, replaces [IRKindIdent] with
-	// checksum values if non-nil.
+	// checksum values if non-nil. The pathname field may not be populated.
 	inputs map[Artifact]cureRes
 }
 
@@ -173,7 +173,7 @@ func (i *IContext) mustWrite(p []byte) {
 
 // WriteIdent writes the identifier of [Artifact] to the IR. The behaviour of
 // WriteIdent is not defined for an [Artifact] not part of the slice returned by
-// [Artifact.Dependencies].
+// [Artifact.Inputs].
 func (i *IContext) WriteIdent(a Artifact) {
 	buf := i.ic.getIdentBuf()
 	defer i.ic.putIdentBuf(buf)
@@ -235,7 +235,8 @@ func (ic *irCache) Encode(w io.Writer, a Artifact) (err error) {
 }
 
 // encode implements Encode but replaces identifiers with their cured checksums
-// for a non-nil ident. Caller must acquire Cache.identMu.
+// for a non-nil ident. The pathname field is unused. Caller must acquire
+// Cache.identMu.
 func (ic *irCache) encode(
 	w io.Writer,
 	a Artifact,
@@ -432,9 +433,9 @@ type (
 		// Common buffer for word-sized reads.
 		buf [wordSize]byte
 
-		// Dependencies sent before params, sorted by identifier. Resliced on
+		// Inputs sent before params, sorted by identifier. Resliced on
 		// each call to Next and checked to be depleted during Finalise.
-		deps []*extIdent
+		inputs []*extIdent
 
 		// Number of values already read, -1 denotes a finalised IRReader.
 		count int
@@ -518,7 +519,7 @@ const (
 	irMaxValues = 1 << 12
 
 	// irMaxDeps is the arbitrary maximum number of direct dependencies allowed
-	// to be returned by [Artifact.Dependencies] and subsequently decoded by
+	// to be returned by [Artifact.Inputs] and subsequently decoded by
 	// [IRDecoder].
 	irMaxDeps = 1 << 10
 )
@@ -603,17 +604,17 @@ func (ir *IRReader) mustReadHeader(k IRValueKind) {
 
 // putAll returns all dependency buffers to the underlying [Cache].
 func (ir *IRReader) putAll() {
-	for _, buf := range ir.deps {
+	for _, buf := range ir.inputs {
 		ir.d.c.putIdentBuf(buf)
 	}
-	ir.deps = nil
+	ir.inputs = nil
 }
 
 // DiscardAll discards all unstructured dependencies. This is useful to
 // implementations that encode dependencies as [IRKindIdent] which are read back
 // via ReadIdent.
 func (ir *IRReader) DiscardAll() {
-	if ir.deps == nil {
+	if ir.inputs == nil {
 		panic("attempting to discard dependencies twice")
 	}
 	ir.putAll()
@@ -625,13 +626,13 @@ var ErrDependencyDepleted = errors.New("reading past end of dependencies")
 
 // Next returns the next unstructured dependency.
 func (ir *IRReader) Next() Artifact {
-	if len(ir.deps) == 0 {
+	if len(ir.inputs) == 0 {
 		panic(ErrDependencyDepleted)
 	}
 
-	id := unique.Make(ID(ir.deps[0][wordSize:]))
+	id := unique.Make(ID(ir.inputs[0][wordSize:]))
-	ir.d.c.putIdentBuf(ir.deps[0])
+	ir.d.c.putIdentBuf(ir.inputs[0])
-	ir.deps = ir.deps[1:]
+	ir.inputs = ir.inputs[1:]
 
 	if a, ok := ir.d.ident[id]; !ok {
 		ir.putAll()
@@ -684,8 +685,8 @@ func (ir *IRReader) Finalise() (checksum unique.Handle[Checksum], ok bool) {
 	}
 	ir.r, ir.ibw = nil, nil
 
-	if len(ir.deps) != 0 {
+	if len(ir.inputs) != 0 {
-		panic(MissedDependencyError(len(ir.deps)))
+		panic(MissedDependencyError(len(ir.inputs)))
 	}
 
 	return
@@ -778,11 +779,11 @@ func (d *IRDecoder) decode() (a Artifact, err error) {
 		err = ErrIRDepend
 		return
 	}
-	ir.deps = make([]*extIdent, sz)
+	ir.inputs = make([]*extIdent, sz)
-	for i := range ir.deps {
+	for i := range ir.inputs {
-		ir.deps[i] = d.c.getIdentBuf()
+		ir.inputs[i] = d.c.getIdentBuf()
 	}
-	for _, buf := range ir.deps {
+	for _, buf := range ir.inputs {
 		ir.mustRead(buf[:])
 	}
 

internal/pkg/pkg.go

@@ -368,7 +368,7 @@ var _ error = InvalidLookupError{}
 
 // GetArtifact returns the identifier pathname and checksum of an [Artifact].
 // Calling Pathname with an [Artifact] not part of the slice returned by
-// [Artifact.Dependencies] panics.
+// [Artifact.Inputs] panics.
 func (f *FContext) GetArtifact(a Artifact) (
 	pathname *check.Absolute,
 	checksum unique.Handle[Checksum],
@@ -678,6 +678,10 @@ const (
 
 	// CIgnoreSubstitutes disables content-based dependency substitution.
 	CIgnoreSubstitutes
+
+	// CExternShallow arranges for only non-flood inputs to be fetched when
+	// curing an [Artifact] available via the external cache.
+	CExternShallow
 )
 
 // toplevel holds [context.WithCancel] over caller-supplied context, where all
@@ -1914,25 +1918,57 @@ func (c *Cache) tryExtern(ctx context.Context, id unique.Handle[ID]) (
 }
 
 // cureMany concurrently collects outcome of multiple [Artifact].
-func (c *Cache) cureMany(inputs []Artifact, r map[Artifact]cureRes) error {
+func (c *Cache) cureMany(
+	inputs []Artifact,
+	r map[Artifact]cureRes,
+	shallow bool,
+) ([]bool, error) {
 	var wg sync.WaitGroup
 	wg.Add(len(inputs))
+	var mask []bool
 	res := make([]cureRes, len(inputs))
 	errs := make(DependencyCureError, 0, len(inputs))
 	var errsMu sync.Mutex
+	if shallow {
+		mask = make([]bool, len(inputs))
+	}
 	for i, d := range inputs {
+		if shallow {
+			if _, ok := d.(FloodArtifact); ok {
+				mask[i] = true
+				wg.Done()
+				continue
+			}
+
+			if kc, ok := d.(KnownChecksum); ok {
+				res[i].checksum = unique.Make(kc.Checksum())
+				wg.Done()
+				continue
+			}
+		}
 		pending := pendingArtifactDep{d, &res[i], &errs, &errsMu, &wg}
 		go pending.cure(c)
 	}
 	wg.Wait()
 
 	if len(errs) > 0 {
-		return &errs
+		return mask, &errs
 	}
 	for i, p := range res {
+		if shallow && mask[i] {
+			continue
+		}
 		r[inputs[i]] = p
 	}
-	return nil
+	return mask, nil
+}
+
+// HangingInputError describes an input of an [Artifact] on a sparse cache
+// without an outcome available locally or via [External].
+type HangingInputError unique.Handle[ID]
+
+func (e HangingInputError) Error() string {
+	return Encode(unique.Handle[ID](e).Value()) + " is unavailable"
 }
 
 // cure implements Cure without acquiring a read lock on abortMu. cure must not
@@ -2193,12 +2229,62 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 		break
 
 	case FloodArtifact:
+		var externChecksum unique.Handle[Checksum]
+		if externChecksum, err = c.tryExtern(ctx, id); err != nil {
+			if c.msg.IsVerbose() {
+				c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
+			}
+			return
+		}
+		extern := externChecksum != zeroChecksum
+		shallow := extern && c.flags&CExternShallow != 0
+
 		inputs := a.Inputs()
 		f := FContext{t, make(map[Artifact]cureRes, len(inputs))}
-		if err = c.cureMany(inputs, f.inputs); err != nil {
+		var mask []bool
+		if mask, err = c.cureMany(inputs, f.inputs, shallow); err != nil {
 			return
 		}
 
+		if shallow {
+			for i, d := range inputs {
+				if !mask[i] {
+					continue
+				}
+
+				if kc, ok := d.(KnownChecksum); ok {
+					f.inputs[d] = cureRes{checksum: unique.Make(kc.Checksum())}
+					continue
+				}
+
+				var sum unique.Handle[Checksum]
+				did := c.Ident(d)
+				sum, err = c.tryLocal(did)
+				if err != nil {
+					if !errors.Is(err, os.ErrNotExist) {
+						return
+					}
+
+					sum, err = c.tryExtern(ctx, did)
+					if err != nil {
+						return
+					}
+					if sum == zeroChecksum {
+						if c.msg.IsVerbose() {
+							c.msg.Verbosef(
+								"input %s not available",
+								reportName(d, did),
+							)
+						}
+						err = HangingInputError(did)
+						return
+					}
+				}
+
+				f.inputs[d] = cureRes{checksum: sum}
+			}
+		}
+
 		sh := sha512.New384()
 		err = c.encode(sh, a, f.inputs)
 		if err != nil {
@@ -2245,14 +2331,7 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 
 		defer f.destroy(&err)
 
-		var externChecksum unique.Handle[Checksum]
+		if extern {
-		if externChecksum, err = c.tryExtern(ctx, id); err != nil {
-			if c.msg.IsVerbose() {
-				c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
-			}
-			return
-		}
-		if externChecksum != zeroChecksum {
 			if checksum != zeroChecksum && externChecksum != checksum {
 				err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()}
 				if c.msg.IsVerbose() {

internal/pkg/pkg_test.go

@@ -1621,6 +1621,152 @@ func TestCache(t *testing.T) {
 
 			"work": {Mode: fs.ModeDir | 0700},
 		}},
+
+		{"extern shallow", pkg.CExternShallow, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+			a := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("extern"),
+			}
+			wantIdent := c.Ident(a)
+			wantOutput := expectsFS{
+				".":      {Mode: fs.ModeDir | 0500},
+				"result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+			}
+			var wantChecksum pkg.Checksum
+			if err := pkg.SumFS(
+				&wantChecksum,
+				fstest.MapFS(wantOutput),
+				".",
+			); err != nil {
+				t.Fatal(err)
+			}
+			wantChecksumH := unique.Make(wantChecksum)
+
+			_a := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("extern substitute"),
+				deps:   []pkg.Artifact{pkg.NewFile("", nil)},
+			}
+			_wantIdent := c.Ident(_a)
+			_wantOutput := expectsFS{
+				".": {Mode: fs.ModeDir | 0500},
+			}
+			var _wantChecksum pkg.Checksum
+			if err := pkg.SumFS(
+				&_wantChecksum,
+				fstest.MapFS(_wantOutput),
+				".",
+			); err != nil {
+				t.Fatal(err)
+			}
+			_wantChecksumH := unique.Make(_wantChecksum)
+
+			kca := pkg.NewExec(
+				"", "",
+				new(pkg.Checksum), 0, false, false,
+				fhs.AbsRoot, nil, fhs.AbsRoot, nil,
+			)
+			kcIdent := c.Ident(kca)
+
+			fia := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("flood input"),
+				deps:   []pkg.Artifact{_a},
+			}
+			fiaIdent := c.Ident(fia)
+
+			flia := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("flood input local"),
+				deps:   []pkg.Artifact{_a},
+			}
+			fliaIdent := c.Ident(flia)
+
+			h := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("hanging input"),
+			}
+
+			fhia := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("flood input local"),
+				deps:   []pkg.Artifact{h},
+			}
+			fhiaIdent := c.Ident(fhia)
+
+			c.SetExternal(stubExtern{
+				artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{
+					wantIdent:  wantChecksum,
+					_wantIdent: _wantChecksum,
+					kcIdent:    wantChecksum,
+					fiaIdent:   wantChecksum,
+					fliaIdent:  wantChecksum,
+					fhiaIdent:  wantChecksum,
+				},
+				checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{
+					wantChecksumH:  fstest.MapFS(wantOutput),
+					_wantChecksumH: fstest.MapFS(_wantOutput),
+				},
+				status: map[unique.Handle[pkg.ID]]string{
+					wantIdent: "\x00",
+					kcIdent:   "unreachable",
+				},
+			})
+
+			cureMany(t, c, []cureStep{
+				{"extern", a, base.Append(
+					"identifier",
+					pkg.Encode(wantIdent.Value()),
+				), wantOutput, nil},
+
+				{"flood shallow extern", fia, base.Append(
+					"identifier",
+					pkg.Encode(fiaIdent.Value()),
+				), wantOutput, nil},
+
+				{"substitute", _a, base.Append(
+					"identifier",
+					pkg.Encode(_wantIdent.Value()),
+				), _wantOutput, nil},
+
+				{"flood shallow local", flia, base.Append(
+					"identifier",
+					pkg.Encode(fliaIdent.Value()),
+				), wantOutput, nil},
+
+				{"flood shallow hanging", fhia, nil, nil,
+					pkg.HangingInputError(c.Ident(h))},
+
+				{"mismatch", kca, nil, nil, &pkg.ChecksumMismatchError{
+					Got: wantChecksum,
+				}},
+			})
+		}, expectsFS{
+			".": {Mode: fs.ModeDir | 0700},
+
+			"checksum": {Mode: fs.ModeDir | 0700},
+			"checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU":        {Mode: fs.ModeDir | 0500},
+			"checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl":        {Mode: fs.ModeDir | 0500},
+			"checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+
+			"identifier": {Mode: fs.ModeDir | 0700},
+			"identifier/4HqRo4uTwRQjfy3d2cujMoDC_pC4iv20h4a7NYlx0UdbVuky18o5iK78TFEfPX2U": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			"identifier/7AZcJm58ghFyTVf_v2baSntgpsxkP5el7ti9dC77C29n8YTEqQW9jRW92KGNdYnz": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+			"identifier/RJbTOOIlqsEkAxGlTIsbp4mHPO0Q2IlwU7Gh_FIaTxnhgxUjAQPXXgPyP2GJWX2A": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+			"identifier/c4aCI00C-ZVyo_FQDQLl1OYK4U_kjzxwrLdFDiXMHnbMcZXCkXo_nxUWauScZ_4V": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			"identifier/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+			"identifier/lqjQtzpJ6asInpsdxqNYVjTkRHK-gW0eUCqNCCk1KhZTuKwKAc5Z0H7tS-GWfRlg": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+			"status": {Mode: fs.ModeDir | 0700},
+			"status/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: 0400, Data: []byte("\x00")},
+
+			"substitute": {Mode: fs.ModeDir | 0700},
+			"substitute/qOYrxy9ztKeOA96Os811_0Ox5sd8FBOxis6psJAnRJL5MLazFMaqmd4g7t7k1OHk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			"substitute/vLTXKbImP9XTTSi7P3PIuqy5L5UgX4dFQm2traX4X_pCDvg9GDraX1LmaUpgLu4w": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+			"substitute/yn1xEs5uSnu5EFLtkvjFSQ0C_fv50hU_w2wfDH_WBEzQhUiPJfyS3Eegy6YSclYN": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+			"work": {Mode: fs.ModeDir | 0700},
+		}},
 	}
 	checkWithCache(t, testCases)
 }