internal/pkg: replace outcomes from external cache

This is primarily useful for implementing a mirror service. This change also works around the zero-dependency FloodArtifact edge case.

Signed-off-by: Ophestra <cat@gensokyo.uk>

internal/pkg/pkg.go

@@ -187,7 +187,7 @@ func makeStatusHeader(extension string) string {
 var statusHeader = makeStatusHeader("")
 
 // prepareStatus initialises the status file once.
-func (t *TContext) prepareStatus() error {
+func (t *TContext) prepareStatus(writeHeader bool) error {
 	if t.statusPath != nil || t.status != nil {
 		return t.statusErr
 	}
@@ -204,14 +204,16 @@ func (t *TContext) prepareStatus() error {
 		return t.statusErr
 	}
 
-	_, t.statusErr = t.status.WriteString(statusHeader)
+	if writeHeader {
+		_, t.statusErr = t.status.WriteString(statusHeader)
+	}
 	return t.statusErr
 }
 
 // GetStatusWriter returns a [io.Writer] for build logs. The caller must not
 // seek this writer before the position it was first returned in.
 func (t *TContext) GetStatusWriter() (io.Writer, error) {
-	err := t.prepareStatus()
+	err := t.prepareStatus(true)
 	return t.status, err
 }
 
@@ -332,6 +334,28 @@ type FContext struct {
 	deps map[Artifact]cureRes
 }
 
+// linkSubstitute links status for substitute if populated.
+func (f *FContext) linkSubstitute(ids, substitutes string) (err error) {
+	if f.status == nil || ids == substitutes {
+		return
+	}
+
+	statusS := f.cache.base.Append(
+		dirStatus,
+		substitutes,
+	)
+	f.cache.checksumMu.Lock()
+	err = os.Link(f.cache.base.Append(
+		dirStatus,
+		ids,
+	).String(), statusS.String())
+	f.cache.checksumMu.Unlock()
+	if err == nil {
+		f.statusSPath = statusS
+	}
+	return
+}
+
 // InvalidLookupError is the identifier of non-dependency [Artifact] looked up
 // via [FContext.GetArtifact] by a misbehaving [Artifact] implementation.
 type InvalidLookupError ID
@@ -670,6 +694,20 @@ type pendingCure struct {
 	cancel context.CancelFunc
 }
 
+// An External cache provides prepared [Artifact] cure outcomes.
+type External interface {
+	// Artifact returns the address of the [Checksum] of the cure outcome of
+	// an [Artifact] corresponding to id, or nil if this [Artifact] is not
+	// available in the external cache.
+	Artifact(id unique.Handle[ID]) (*Checksum, error)
+	// Checksum returns an [Artifact] producing the specified checksum.
+	Checksum(checksum unique.Handle[Checksum]) Artifact
+	// Status returns [io.ReadCloser] of the status file of an [Artifact]
+	// corresponding to id, or nil if this [Artifact] is not available or a
+	// status file is not present.
+	Status(id unique.Handle[ID]) (io.ReadCloser, error)
+}
+
 // Cache is a support layer that implementations of [Artifact] can use to store
 // cured [Artifact] data in a content addressed fashion.
 type Cache struct {
@@ -720,6 +758,11 @@ type Cache struct {
 	// Buffered I/O free list, must not be accessed directly.
 	brPool, bwPool sync.Pool
 
+	// Optional external cache implementation.
+	extern External
+	// Synchronises access to extern.
+	externMu sync.RWMutex
+
 	// Unlocks the on-filesystem cache. Must only be called from Close.
 	unlock func()
 	// Whether [Cache] is considered closed.
@@ -874,6 +917,13 @@ func readlinkChecksum(a *check.Absolute, buf *Checksum) error {
 	return Decode(buf, linkname[len(checksumLinknamePrefix):])
 }
 
+// SetExternal sets e as the [External] implementation of c.
+func (c *Cache) SetExternal(e External) {
+	c.externMu.Lock()
+	c.extern = e
+	c.externMu.Unlock()
+}
+
 // ScrubError describes the outcome of a [Cache.Scrub] call where errors were
 // found and removed from the underlying storage of [Cache].
 type ScrubError struct {
@@ -1782,6 +1832,40 @@ func (r *RContext) NewMeasuredReader(
 	return r.cache.newMeasuredReader(rc, checksum)
 }
 
+// tryExtern attempts to obtain an [Artifact] outcome from extern.
+func (c *Cache) tryExtern(id unique.Handle[ID]) (
+	unique.Handle[Checksum],
+	io.ReadCloser,
+	error,
+) {
+	c.externMu.RLock()
+	defer c.externMu.RUnlock()
+
+	if c.extern == nil {
+		return zeroChecksum, nil, nil
+	}
+
+	v, err := c.extern.Artifact(id)
+	if err != nil {
+		return zeroChecksum, nil, err
+	}
+	if v == nil {
+		return zeroChecksum, nil, nil
+	}
+	checksum := unique.Make(*v)
+
+	var got unique.Handle[Checksum]
+	if _, got, err = c.Cure(c.extern.Checksum(checksum)); err != nil {
+		return checksum, nil, err
+	} else if got != checksum {
+		return zeroChecksum, nil, &ChecksumMismatchError{got.Value(), checksum.Value()}
+	}
+
+	var status io.ReadCloser
+	status, err = c.extern.Status(id)
+	return checksum, status, err
+}
+
 // cure implements Cure without acquiring a read lock on abortMu. cure must not
 // be entered during Abort.
 func (c *Cache) cure(a Artifact, curesExempt bool) (
@@ -1848,7 +1932,7 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 				err = zeroTimes(pathname.String())
 			}
 
-			if err == nil && alternative != nil {
+			if err == nil && alternative != nil && substitute != id {
 				c.substituteMu.Lock()
 				err = os.Symlink(
 					linkname,
@@ -2116,26 +2200,60 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 		}
 
 		defer f.destroy(&err)
+
+		var (
+			externChecksum unique.Handle[Checksum]
+			externStatus   io.ReadCloser
+		)
+		if externChecksum, externStatus, err = c.tryExtern(id); err != nil {
+			if c.msg.IsVerbose() {
+				c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
+			}
+			return
+		}
+		if externChecksum != zeroChecksum {
+			if checksum != zeroChecksum && externChecksum != checksum {
+				if externStatus != nil {
+					_ = externStatus.Close()
+				}
+				err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()}
+				if c.msg.IsVerbose() {
+					c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
+				}
+				return
+			}
+
+			checksum = externChecksum
+			checksums = Encode(checksum.Value())
+			checksumPathname = c.base.Append(
+				dirChecksum,
+				checksums,
+			)
+
+			if externStatus != nil {
+				if err = f.prepareStatus(false); err != nil {
+					_ = externStatus.Close()
+					return
+				} else if _, err = io.Copy(f.status, externStatus); err != nil {
+					_ = externStatus.Close()
+					return
+				} else if err = externStatus.Close(); err != nil {
+					return
+				} else if err = f.linkSubstitute(ids, substitutes); err != nil {
+					return
+				}
+			}
+			return
+		}
+
 		if err = c.enterCure(a, curesExempt); err != nil {
 			return
 		}
 		err = ca.Cure(&f)
 		c.exitCure(a, curesExempt)
 
-		if err == nil && f.status != nil {
+		if err == nil {
-			statusS := c.base.Append(
+			err = f.linkSubstitute(ids, substitutes)
-				dirStatus,
-				substitutes,
-			)
-			c.checksumMu.Lock()
-			err = os.Link(c.base.Append(
-				dirStatus,
-				ids,
-			).String(), statusS.String())
-			c.checksumMu.Unlock()
-			if err == nil {
-				f.statusSPath = statusS
-			}
 		}
 		if err != nil {
 			if c.msg.IsVerbose() {

internal/pkg/pkg_test.go

@@ -197,6 +197,35 @@ func newStubFile(
 	}
 }
 
+// stubExtern implements [External] with hardcoded prepared outcomes.
+type stubExtern struct {
+	artifact map[unique.Handle[pkg.ID]]pkg.Checksum
+	checksum map[unique.Handle[pkg.Checksum]]fstest.MapFS
+	status   map[unique.Handle[pkg.ID]]string
+}
+
+func (e stubExtern) Artifact(id unique.Handle[pkg.ID]) (*pkg.Checksum, error) {
+	if checksum, ok := e.artifact[id]; ok {
+		return &checksum, nil
+	}
+	return nil, nil
+}
+
+func (e stubExtern) Checksum(checksum unique.Handle[pkg.Checksum]) pkg.Artifact {
+	var buf bytes.Buffer
+	if err := pkg.Write(e.checksum[checksum], ".", &buf); err != nil {
+		panic(err)
+	}
+	return pkg.NewArchive(pkg.NewFile("", buf.Bytes()))
+}
+
+func (e stubExtern) Status(id unique.Handle[pkg.ID]) (io.ReadCloser, error) {
+	if status, ok := e.status[id]; ok {
+		return io.NopCloser(strings.NewReader(status)), nil
+	}
+	return nil, nil
+}
+
 // destroyArtifact removes all traces of an [Artifact] from the on-disk cache.
 // Do not use this in a test case without a very good reason to do so.
 func destroyArtifact(
@@ -1490,6 +1519,108 @@ func TestCache(t *testing.T) {
 
 			"work": {Mode: fs.ModeDir | 0700},
 		}},
+
+		{"extern", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+			a := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("extern"),
+			}
+			wantIdent := c.Ident(a)
+			wantOutput := expectsFS{
+				".":      {Mode: fs.ModeDir | 0500},
+				"result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+			}
+			var wantChecksum pkg.Checksum
+			if err := pkg.SumFS(
+				&wantChecksum,
+				fstest.MapFS(wantOutput),
+				".",
+			); err != nil {
+				t.Fatal(err)
+			}
+			wantChecksumH := unique.Make(wantChecksum)
+
+			_a := &stubArtifactF{
+				kind:   pkg.KindExec,
+				params: []byte("extern substitute"),
+				deps:   []pkg.Artifact{pkg.NewFile("", nil)},
+			}
+			_wantIdent := c.Ident(_a)
+			_wantOutput := expectsFS{
+				".": {Mode: fs.ModeDir | 0500},
+			}
+			var _wantChecksum pkg.Checksum
+			if err := pkg.SumFS(
+				&_wantChecksum,
+				fstest.MapFS(_wantOutput),
+				".",
+			); err != nil {
+				t.Fatal(err)
+			}
+			_wantChecksumH := unique.Make(_wantChecksum)
+
+			kca := pkg.NewExec(
+				"", "",
+				new(pkg.Checksum), 0, false, false,
+				fhs.AbsRoot, nil, fhs.AbsRoot, nil,
+			)
+			kcIdent := c.Ident(kca)
+
+			c.SetExternal(stubExtern{
+				artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{
+					wantIdent:  wantChecksum,
+					_wantIdent: _wantChecksum,
+					kcIdent:    wantChecksum,
+				},
+				checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{
+					wantChecksumH:  fstest.MapFS(wantOutput),
+					_wantChecksumH: fstest.MapFS(_wantOutput),
+				},
+				status: map[unique.Handle[pkg.ID]]string{
+					wantIdent: "\x00",
+					kcIdent:   "unreachable",
+				},
+			})
+
+			cureMany(t, c, []cureStep{
+				{"extern", a, base.Append(
+					"identifier",
+					pkg.Encode(wantIdent.Value()),
+				), wantOutput, nil},
+
+				{"substitute", _a, base.Append(
+					"identifier",
+					pkg.Encode(_wantIdent.Value()),
+				), _wantOutput, nil},
+
+				{"mismatch", kca, nil, nil, &pkg.ChecksumMismatchError{
+					Got: wantChecksum,
+				}},
+			})
+		}, expectsFS{
+			".": {Mode: fs.ModeDir | 0700},
+
+			"checksum": {Mode: fs.ModeDir | 0700},
+			"checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU":        {Mode: fs.ModeDir | 0500},
+			"checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb":        {Mode: 0400},
+			"checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl":        {Mode: fs.ModeDir | 0500},
+			"checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+
+			"identifier": {Mode: fs.ModeDir | 0700},
+			"identifier/4HqRo4uTwRQjfy3d2cujMoDC_pC4iv20h4a7NYlx0UdbVuky18o5iK78TFEfPX2U": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			"identifier/7AZcJm58ghFyTVf_v2baSntgpsxkP5el7ti9dC77C29n8YTEqQW9jRW92KGNdYnz": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+			"identifier/c4aCI00C-ZVyo_FQDQLl1OYK4U_kjzxwrLdFDiXMHnbMcZXCkXo_nxUWauScZ_4V": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			"identifier/cNoG77frXGRCJa7fUi1INKUEQg7L4qrX5acsSv-wqZdGZT7dQwM93rD3at6kSFFF": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+			"identifier/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+			"status": {Mode: fs.ModeDir | 0700},
+			"status/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: 0400, Data: []byte("\x00")},
+
+			"substitute": {Mode: fs.ModeDir | 0700},
+			"substitute/qOYrxy9ztKeOA96Os811_0Ox5sd8FBOxis6psJAnRJL5MLazFMaqmd4g7t7k1OHk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+			"work": {Mode: fs.ModeDir | 0700},
+		}},
 	}
 	checkWithCache(t, testCases)
 }