diff --git a/internal/pkg/ir.go b/internal/pkg/ir.go index 35fe76e7..5a7e7f56 100644 --- a/internal/pkg/ir.go +++ b/internal/pkg/ir.go @@ -90,7 +90,7 @@ type IContext struct { // returns and must not be exposed directly. w io.Writer // Optional [Artifact] to cureRes cache, replaces [IRKindIdent] with - // checksum values if non-nil. + // checksum values if non-nil. The pathname field may not be populated. inputs map[Artifact]cureRes } @@ -173,7 +173,7 @@ func (i *IContext) mustWrite(p []byte) { // WriteIdent writes the identifier of [Artifact] to the IR. The behaviour of // WriteIdent is not defined for an [Artifact] not part of the slice returned by -// [Artifact.Dependencies]. +// [Artifact.Inputs]. func (i *IContext) WriteIdent(a Artifact) { buf := i.ic.getIdentBuf() defer i.ic.putIdentBuf(buf) @@ -235,7 +235,8 @@ func (ic *irCache) Encode(w io.Writer, a Artifact) (err error) { } // encode implements Encode but replaces identifiers with their cured checksums -// for a non-nil ident. Caller must acquire Cache.identMu. +// for a non-nil ident. The pathname field is unused. Caller must acquire +// Cache.identMu. func (ic *irCache) encode( w io.Writer, a Artifact, @@ -432,9 +433,9 @@ type ( // Common buffer for word-sized reads. buf [wordSize]byte - // Dependencies sent before params, sorted by identifier. Resliced on + // Inputs sent before params, sorted by identifier. Resliced on // each call to Next and checked to be depleted during Finalise. - deps []*extIdent + inputs []*extIdent // Number of values already read, -1 denotes a finalised IRReader. count int @@ -518,7 +519,7 @@ const ( irMaxValues = 1 << 12 // irMaxDeps is the arbitrary maximum number of direct dependencies allowed - // to be returned by [Artifact.Dependencies] and subsequently decoded by + // to be returned by [Artifact.Inputs] and subsequently decoded by // [IRDecoder]. irMaxDeps = 1 << 10 ) @@ -603,17 +604,17 @@ func (ir *IRReader) mustReadHeader(k IRValueKind) { // putAll returns all dependency buffers to the underlying [Cache]. func (ir *IRReader) putAll() { - for _, buf := range ir.deps { + for _, buf := range ir.inputs { ir.d.c.putIdentBuf(buf) } - ir.deps = nil + ir.inputs = nil } // DiscardAll discards all unstructured dependencies. This is useful to // implementations that encode dependencies as [IRKindIdent] which are read back // via ReadIdent. func (ir *IRReader) DiscardAll() { - if ir.deps == nil { + if ir.inputs == nil { panic("attempting to discard dependencies twice") } ir.putAll() @@ -625,13 +626,13 @@ var ErrDependencyDepleted = errors.New("reading past end of dependencies") // Next returns the next unstructured dependency. func (ir *IRReader) Next() Artifact { - if len(ir.deps) == 0 { + if len(ir.inputs) == 0 { panic(ErrDependencyDepleted) } - id := unique.Make(ID(ir.deps[0][wordSize:])) - ir.d.c.putIdentBuf(ir.deps[0]) - ir.deps = ir.deps[1:] + id := unique.Make(ID(ir.inputs[0][wordSize:])) + ir.d.c.putIdentBuf(ir.inputs[0]) + ir.inputs = ir.inputs[1:] if a, ok := ir.d.ident[id]; !ok { ir.putAll() @@ -684,8 +685,8 @@ func (ir *IRReader) Finalise() (checksum unique.Handle[Checksum], ok bool) { } ir.r, ir.ibw = nil, nil - if len(ir.deps) != 0 { - panic(MissedDependencyError(len(ir.deps))) + if len(ir.inputs) != 0 { + panic(MissedDependencyError(len(ir.inputs))) } return @@ -778,11 +779,11 @@ func (d *IRDecoder) decode() (a Artifact, err error) { err = ErrIRDepend return } - ir.deps = make([]*extIdent, sz) - for i := range ir.deps { - ir.deps[i] = d.c.getIdentBuf() + ir.inputs = make([]*extIdent, sz) + for i := range ir.inputs { + ir.inputs[i] = d.c.getIdentBuf() } - for _, buf := range ir.deps { + for _, buf := range ir.inputs { ir.mustRead(buf[:]) } diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go index 78e62e85..aadfa308 100644 --- a/internal/pkg/pkg.go +++ b/internal/pkg/pkg.go @@ -368,7 +368,7 @@ var _ error = InvalidLookupError{} // GetArtifact returns the identifier pathname and checksum of an [Artifact]. // Calling Pathname with an [Artifact] not part of the slice returned by -// [Artifact.Dependencies] panics. +// [Artifact.Inputs] panics. func (f *FContext) GetArtifact(a Artifact) ( pathname *check.Absolute, checksum unique.Handle[Checksum], @@ -678,6 +678,10 @@ const ( // CIgnoreSubstitutes disables content-based dependency substitution. CIgnoreSubstitutes + + // CExternShallow arranges for only non-flood inputs to be fetched when + // curing an [Artifact] available via the external cache. + CExternShallow ) // toplevel holds [context.WithCancel] over caller-supplied context, where all @@ -1914,25 +1918,57 @@ func (c *Cache) tryExtern(ctx context.Context, id unique.Handle[ID]) ( } // cureMany concurrently collects outcome of multiple [Artifact]. -func (c *Cache) cureMany(inputs []Artifact, r map[Artifact]cureRes) error { +func (c *Cache) cureMany( + inputs []Artifact, + r map[Artifact]cureRes, + shallow bool, +) ([]bool, error) { var wg sync.WaitGroup wg.Add(len(inputs)) + var mask []bool res := make([]cureRes, len(inputs)) errs := make(DependencyCureError, 0, len(inputs)) var errsMu sync.Mutex + if shallow { + mask = make([]bool, len(inputs)) + } for i, d := range inputs { + if shallow { + if _, ok := d.(FloodArtifact); ok { + mask[i] = true + wg.Done() + continue + } + + if kc, ok := d.(KnownChecksum); ok { + res[i].checksum = unique.Make(kc.Checksum()) + wg.Done() + continue + } + } pending := pendingArtifactDep{d, &res[i], &errs, &errsMu, &wg} go pending.cure(c) } wg.Wait() if len(errs) > 0 { - return &errs + return mask, &errs } for i, p := range res { + if shallow && mask[i] { + continue + } r[inputs[i]] = p } - return nil + return mask, nil +} + +// HangingInputError describes an input of an [Artifact] on a sparse cache +// without an outcome available locally or via [External]. +type HangingInputError unique.Handle[ID] + +func (e HangingInputError) Error() string { + return Encode(unique.Handle[ID](e).Value()) + " is unavailable" } // cure implements Cure without acquiring a read lock on abortMu. cure must not @@ -2193,12 +2229,62 @@ func (c *Cache) cure(a Artifact, curesExempt bool) ( break case FloodArtifact: + var externChecksum unique.Handle[Checksum] + if externChecksum, err = c.tryExtern(ctx, id); err != nil { + if c.msg.IsVerbose() { + c.msg.Verbosef("extern %s: %v", reportName(ca, id), err) + } + return + } + extern := externChecksum != zeroChecksum + shallow := extern && c.flags&CExternShallow != 0 + inputs := a.Inputs() f := FContext{t, make(map[Artifact]cureRes, len(inputs))} - if err = c.cureMany(inputs, f.inputs); err != nil { + var mask []bool + if mask, err = c.cureMany(inputs, f.inputs, shallow); err != nil { return } + if shallow { + for i, d := range inputs { + if !mask[i] { + continue + } + + if kc, ok := d.(KnownChecksum); ok { + f.inputs[d] = cureRes{checksum: unique.Make(kc.Checksum())} + continue + } + + var sum unique.Handle[Checksum] + did := c.Ident(d) + sum, err = c.tryLocal(did) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return + } + + sum, err = c.tryExtern(ctx, did) + if err != nil { + return + } + if sum == zeroChecksum { + if c.msg.IsVerbose() { + c.msg.Verbosef( + "input %s not available", + reportName(d, did), + ) + } + err = HangingInputError(did) + return + } + } + + f.inputs[d] = cureRes{checksum: sum} + } + } + sh := sha512.New384() err = c.encode(sh, a, f.inputs) if err != nil { @@ -2245,14 +2331,7 @@ func (c *Cache) cure(a Artifact, curesExempt bool) ( defer f.destroy(&err) - var externChecksum unique.Handle[Checksum] - if externChecksum, err = c.tryExtern(ctx, id); err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef("extern %s: %v", reportName(ca, id), err) - } - return - } - if externChecksum != zeroChecksum { + if extern { if checksum != zeroChecksum && externChecksum != checksum { err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()} if c.msg.IsVerbose() { diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go index 01cd2617..d03216f6 100644 --- a/internal/pkg/pkg_test.go +++ b/internal/pkg/pkg_test.go @@ -1621,6 +1621,152 @@ func TestCache(t *testing.T) { "work": {Mode: fs.ModeDir | 0700}, }}, + + {"extern shallow", pkg.CExternShallow, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { + a := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("extern"), + } + wantIdent := c.Ident(a) + wantOutput := expectsFS{ + ".": {Mode: fs.ModeDir | 0500}, + "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, + } + var wantChecksum pkg.Checksum + if err := pkg.SumFS( + &wantChecksum, + fstest.MapFS(wantOutput), + ".", + ); err != nil { + t.Fatal(err) + } + wantChecksumH := unique.Make(wantChecksum) + + _a := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("extern substitute"), + deps: []pkg.Artifact{pkg.NewFile("", nil)}, + } + _wantIdent := c.Ident(_a) + _wantOutput := expectsFS{ + ".": {Mode: fs.ModeDir | 0500}, + } + var _wantChecksum pkg.Checksum + if err := pkg.SumFS( + &_wantChecksum, + fstest.MapFS(_wantOutput), + ".", + ); err != nil { + t.Fatal(err) + } + _wantChecksumH := unique.Make(_wantChecksum) + + kca := pkg.NewExec( + "", "", + new(pkg.Checksum), 0, false, false, + fhs.AbsRoot, nil, fhs.AbsRoot, nil, + ) + kcIdent := c.Ident(kca) + + fia := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("flood input"), + deps: []pkg.Artifact{_a}, + } + fiaIdent := c.Ident(fia) + + flia := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("flood input local"), + deps: []pkg.Artifact{_a}, + } + fliaIdent := c.Ident(flia) + + h := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("hanging input"), + } + + fhia := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("flood input local"), + deps: []pkg.Artifact{h}, + } + fhiaIdent := c.Ident(fhia) + + c.SetExternal(stubExtern{ + artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{ + wantIdent: wantChecksum, + _wantIdent: _wantChecksum, + kcIdent: wantChecksum, + fiaIdent: wantChecksum, + fliaIdent: wantChecksum, + fhiaIdent: wantChecksum, + }, + checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{ + wantChecksumH: fstest.MapFS(wantOutput), + _wantChecksumH: fstest.MapFS(_wantOutput), + }, + status: map[unique.Handle[pkg.ID]]string{ + wantIdent: "\x00", + kcIdent: "unreachable", + }, + }) + + cureMany(t, c, []cureStep{ + {"extern", a, base.Append( + "identifier", + pkg.Encode(wantIdent.Value()), + ), wantOutput, nil}, + + {"flood shallow extern", fia, base.Append( + "identifier", + pkg.Encode(fiaIdent.Value()), + ), wantOutput, nil}, + + {"substitute", _a, base.Append( + "identifier", + pkg.Encode(_wantIdent.Value()), + ), _wantOutput, nil}, + + {"flood shallow local", flia, base.Append( + "identifier", + pkg.Encode(fliaIdent.Value()), + ), wantOutput, nil}, + + {"flood shallow hanging", fhia, nil, nil, + pkg.HangingInputError(c.Ident(h))}, + + {"mismatch", kca, nil, nil, &pkg.ChecksumMismatchError{ + Got: wantChecksum, + }}, + }) + }, expectsFS{ + ".": {Mode: fs.ModeDir | 0700}, + + "checksum": {Mode: fs.ModeDir | 0700}, + "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, + "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500}, + "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, + + "identifier": {Mode: fs.ModeDir | 0700}, + "identifier/4HqRo4uTwRQjfy3d2cujMoDC_pC4iv20h4a7NYlx0UdbVuky18o5iK78TFEfPX2U": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, + "identifier/7AZcJm58ghFyTVf_v2baSntgpsxkP5el7ti9dC77C29n8YTEqQW9jRW92KGNdYnz": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + "identifier/RJbTOOIlqsEkAxGlTIsbp4mHPO0Q2IlwU7Gh_FIaTxnhgxUjAQPXXgPyP2GJWX2A": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + "identifier/c4aCI00C-ZVyo_FQDQLl1OYK4U_kjzxwrLdFDiXMHnbMcZXCkXo_nxUWauScZ_4V": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, + "identifier/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + "identifier/lqjQtzpJ6asInpsdxqNYVjTkRHK-gW0eUCqNCCk1KhZTuKwKAc5Z0H7tS-GWfRlg": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + + "status": {Mode: fs.ModeDir | 0700}, + "status/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: 0400, Data: []byte("\x00")}, + + "substitute": {Mode: fs.ModeDir | 0700}, + "substitute/qOYrxy9ztKeOA96Os811_0Ox5sd8FBOxis6psJAnRJL5MLazFMaqmd4g7t7k1OHk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, + "substitute/vLTXKbImP9XTTSi7P3PIuqy5L5UgX4dFQm2traX4X_pCDvg9GDraX1LmaUpgLu4w": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + "substitute/yn1xEs5uSnu5EFLtkvjFSQ0C_fv50hU_w2wfDH_WBEzQhUiPJfyS3Eegy6YSclYN": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + + "work": {Mode: fs.ModeDir | 0700}, + }}, } checkWithCache(t, testCases) }