diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go index f5b586ee..6542741f 100644 --- a/internal/pkg/pkg.go +++ b/internal/pkg/pkg.go @@ -187,7 +187,7 @@ func makeStatusHeader(extension string) string { var statusHeader = makeStatusHeader("") // prepareStatus initialises the status file once. -func (t *TContext) prepareStatus() error { +func (t *TContext) prepareStatus(writeHeader bool) error { if t.statusPath != nil || t.status != nil { return t.statusErr } @@ -204,14 +204,16 @@ func (t *TContext) prepareStatus() error { return t.statusErr } - _, t.statusErr = t.status.WriteString(statusHeader) + if writeHeader { + _, t.statusErr = t.status.WriteString(statusHeader) + } return t.statusErr } // GetStatusWriter returns a [io.Writer] for build logs. The caller must not // seek this writer before the position it was first returned in. func (t *TContext) GetStatusWriter() (io.Writer, error) { - err := t.prepareStatus() + err := t.prepareStatus(true) return t.status, err } @@ -332,6 +334,28 @@ type FContext struct { deps map[Artifact]cureRes } +// linkSubstitute links status for substitute if populated. +func (f *FContext) linkSubstitute(ids, substitutes string) (err error) { + if f.status == nil || ids == substitutes { + return + } + + statusS := f.cache.base.Append( + dirStatus, + substitutes, + ) + f.cache.checksumMu.Lock() + err = os.Link(f.cache.base.Append( + dirStatus, + ids, + ).String(), statusS.String()) + f.cache.checksumMu.Unlock() + if err == nil { + f.statusSPath = statusS + } + return +} + // InvalidLookupError is the identifier of non-dependency [Artifact] looked up // via [FContext.GetArtifact] by a misbehaving [Artifact] implementation. type InvalidLookupError ID @@ -670,6 +694,20 @@ type pendingCure struct { cancel context.CancelFunc } +// An External cache provides prepared [Artifact] cure outcomes. +type External interface { + // Artifact returns the address of the [Checksum] of the cure outcome of + // an [Artifact] corresponding to id, or nil if this [Artifact] is not + // available in the external cache. + Artifact(id unique.Handle[ID]) (*Checksum, error) + // Checksum returns an [Artifact] producing the specified checksum. + Checksum(checksum unique.Handle[Checksum]) Artifact + // Status returns [io.ReadCloser] of the status file of an [Artifact] + // corresponding to id, or nil if this [Artifact] is not available or a + // status file is not present. + Status(id unique.Handle[ID]) (io.ReadCloser, error) +} + // Cache is a support layer that implementations of [Artifact] can use to store // cured [Artifact] data in a content addressed fashion. type Cache struct { @@ -720,6 +758,11 @@ type Cache struct { // Buffered I/O free list, must not be accessed directly. brPool, bwPool sync.Pool + // Optional external cache implementation. + extern External + // Synchronises access to extern. + externMu sync.RWMutex + // Unlocks the on-filesystem cache. Must only be called from Close. unlock func() // Whether [Cache] is considered closed. @@ -874,6 +917,13 @@ func readlinkChecksum(a *check.Absolute, buf *Checksum) error { return Decode(buf, linkname[len(checksumLinknamePrefix):]) } +// SetExternal sets e as the [External] implementation of c. +func (c *Cache) SetExternal(e External) { + c.externMu.Lock() + c.extern = e + c.externMu.Unlock() +} + // ScrubError describes the outcome of a [Cache.Scrub] call where errors were // found and removed from the underlying storage of [Cache]. type ScrubError struct { @@ -1782,6 +1832,40 @@ func (r *RContext) NewMeasuredReader( return r.cache.newMeasuredReader(rc, checksum) } +// tryExtern attempts to obtain an [Artifact] outcome from extern. +func (c *Cache) tryExtern(id unique.Handle[ID]) ( + unique.Handle[Checksum], + io.ReadCloser, + error, +) { + c.externMu.RLock() + defer c.externMu.RUnlock() + + if c.extern == nil { + return zeroChecksum, nil, nil + } + + v, err := c.extern.Artifact(id) + if err != nil { + return zeroChecksum, nil, err + } + if v == nil { + return zeroChecksum, nil, nil + } + checksum := unique.Make(*v) + + var got unique.Handle[Checksum] + if _, got, err = c.Cure(c.extern.Checksum(checksum)); err != nil { + return checksum, nil, err + } else if got != checksum { + return zeroChecksum, nil, &ChecksumMismatchError{got.Value(), checksum.Value()} + } + + var status io.ReadCloser + status, err = c.extern.Status(id) + return checksum, status, err +} + // cure implements Cure without acquiring a read lock on abortMu. cure must not // be entered during Abort. func (c *Cache) cure(a Artifact, curesExempt bool) ( @@ -1848,7 +1932,7 @@ func (c *Cache) cure(a Artifact, curesExempt bool) ( err = zeroTimes(pathname.String()) } - if err == nil && alternative != nil { + if err == nil && alternative != nil && substitute != id { c.substituteMu.Lock() err = os.Symlink( linkname, @@ -2116,26 +2200,60 @@ func (c *Cache) cure(a Artifact, curesExempt bool) ( } defer f.destroy(&err) + + var ( + externChecksum unique.Handle[Checksum] + externStatus io.ReadCloser + ) + if externChecksum, externStatus, err = c.tryExtern(id); err != nil { + if c.msg.IsVerbose() { + c.msg.Verbosef("extern %s: %v", reportName(ca, id), err) + } + return + } + if externChecksum != zeroChecksum { + if checksum != zeroChecksum && externChecksum != checksum { + if externStatus != nil { + _ = externStatus.Close() + } + err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()} + if c.msg.IsVerbose() { + c.msg.Verbosef("extern %s: %v", reportName(ca, id), err) + } + return + } + + checksum = externChecksum + checksums = Encode(checksum.Value()) + checksumPathname = c.base.Append( + dirChecksum, + checksums, + ) + + if externStatus != nil { + if err = f.prepareStatus(false); err != nil { + _ = externStatus.Close() + return + } else if _, err = io.Copy(f.status, externStatus); err != nil { + _ = externStatus.Close() + return + } else if err = externStatus.Close(); err != nil { + return + } else if err = f.linkSubstitute(ids, substitutes); err != nil { + return + } + } + return + } + if err = c.enterCure(a, curesExempt); err != nil { return } err = ca.Cure(&f) c.exitCure(a, curesExempt) - if err == nil && f.status != nil { - statusS := c.base.Append( - dirStatus, - substitutes, - ) - c.checksumMu.Lock() - err = os.Link(c.base.Append( - dirStatus, - ids, - ).String(), statusS.String()) - c.checksumMu.Unlock() - if err == nil { - f.statusSPath = statusS - } + if err == nil { + err = f.linkSubstitute(ids, substitutes) } if err != nil { if c.msg.IsVerbose() { diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go index 9a397898..1239bed3 100644 --- a/internal/pkg/pkg_test.go +++ b/internal/pkg/pkg_test.go @@ -197,6 +197,35 @@ func newStubFile( } } +// stubExtern implements [External] with hardcoded prepared outcomes. +type stubExtern struct { + artifact map[unique.Handle[pkg.ID]]pkg.Checksum + checksum map[unique.Handle[pkg.Checksum]]fstest.MapFS + status map[unique.Handle[pkg.ID]]string +} + +func (e stubExtern) Artifact(id unique.Handle[pkg.ID]) (*pkg.Checksum, error) { + if checksum, ok := e.artifact[id]; ok { + return &checksum, nil + } + return nil, nil +} + +func (e stubExtern) Checksum(checksum unique.Handle[pkg.Checksum]) pkg.Artifact { + var buf bytes.Buffer + if err := pkg.Write(e.checksum[checksum], ".", &buf); err != nil { + panic(err) + } + return pkg.NewArchive(pkg.NewFile("", buf.Bytes())) +} + +func (e stubExtern) Status(id unique.Handle[pkg.ID]) (io.ReadCloser, error) { + if status, ok := e.status[id]; ok { + return io.NopCloser(strings.NewReader(status)), nil + } + return nil, nil +} + // destroyArtifact removes all traces of an [Artifact] from the on-disk cache. // Do not use this in a test case without a very good reason to do so. func destroyArtifact( @@ -1490,6 +1519,108 @@ func TestCache(t *testing.T) { "work": {Mode: fs.ModeDir | 0700}, }}, + + {"extern", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { + a := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("extern"), + } + wantIdent := c.Ident(a) + wantOutput := expectsFS{ + ".": {Mode: fs.ModeDir | 0500}, + "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, + } + var wantChecksum pkg.Checksum + if err := pkg.SumFS( + &wantChecksum, + fstest.MapFS(wantOutput), + ".", + ); err != nil { + t.Fatal(err) + } + wantChecksumH := unique.Make(wantChecksum) + + _a := &stubArtifactF{ + kind: pkg.KindExec, + params: []byte("extern substitute"), + deps: []pkg.Artifact{pkg.NewFile("", nil)}, + } + _wantIdent := c.Ident(_a) + _wantOutput := expectsFS{ + ".": {Mode: fs.ModeDir | 0500}, + } + var _wantChecksum pkg.Checksum + if err := pkg.SumFS( + &_wantChecksum, + fstest.MapFS(_wantOutput), + ".", + ); err != nil { + t.Fatal(err) + } + _wantChecksumH := unique.Make(_wantChecksum) + + kca := pkg.NewExec( + "", "", + new(pkg.Checksum), 0, false, false, + fhs.AbsRoot, nil, fhs.AbsRoot, nil, + ) + kcIdent := c.Ident(kca) + + c.SetExternal(stubExtern{ + artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{ + wantIdent: wantChecksum, + _wantIdent: _wantChecksum, + kcIdent: wantChecksum, + }, + checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{ + wantChecksumH: fstest.MapFS(wantOutput), + _wantChecksumH: fstest.MapFS(_wantOutput), + }, + status: map[unique.Handle[pkg.ID]]string{ + wantIdent: "\x00", + kcIdent: "unreachable", + }, + }) + + cureMany(t, c, []cureStep{ + {"extern", a, base.Append( + "identifier", + pkg.Encode(wantIdent.Value()), + ), wantOutput, nil}, + + {"substitute", _a, base.Append( + "identifier", + pkg.Encode(_wantIdent.Value()), + ), _wantOutput, nil}, + + {"mismatch", kca, nil, nil, &pkg.ChecksumMismatchError{ + Got: wantChecksum, + }}, + }) + }, expectsFS{ + ".": {Mode: fs.ModeDir | 0700}, + + "checksum": {Mode: fs.ModeDir | 0700}, + "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, + "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400}, + "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500}, + "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, + + "identifier": {Mode: fs.ModeDir | 0700}, + "identifier/4HqRo4uTwRQjfy3d2cujMoDC_pC4iv20h4a7NYlx0UdbVuky18o5iK78TFEfPX2U": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, + "identifier/7AZcJm58ghFyTVf_v2baSntgpsxkP5el7ti9dC77C29n8YTEqQW9jRW92KGNdYnz": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + "identifier/c4aCI00C-ZVyo_FQDQLl1OYK4U_kjzxwrLdFDiXMHnbMcZXCkXo_nxUWauScZ_4V": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, + "identifier/cNoG77frXGRCJa7fUi1INKUEQg7L4qrX5acsSv-wqZdGZT7dQwM93rD3at6kSFFF": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, + "identifier/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, + + "status": {Mode: fs.ModeDir | 0700}, + "status/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: 0400, Data: []byte("\x00")}, + + "substitute": {Mode: fs.ModeDir | 0700}, + "substitute/qOYrxy9ztKeOA96Os811_0Ox5sd8FBOxis6psJAnRJL5MLazFMaqmd4g7t7k1OHk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, + + "work": {Mode: fs.ModeDir | 0700}, + }}, } checkWithCache(t, testCases) }