internal/pkg: expose underlying reader

This will be fully implemented in httpArtifact in a future commit.

Signed-off-by: Ophestra <cat@gensokyo.uk>

internal/pkg/pkg.go

@@ -2,6 +2,7 @@
 package pkg
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/sha512"
@@ -147,14 +148,15 @@ func (t *TContext) GetWorkDir() *check.Absolute { return t.work }
 // create it if they wish to use it, using [os.MkdirAll].
 func (t *TContext) GetTempDir() *check.Absolute { return t.temp }
 
-// Open tries to open [Artifact] for reading. If a implements [File], its data
-// might be used directly, eliminating the roundtrip to vfs. Otherwise, it must
-// cure into a directory containing a single regular file.
+// Open tries to open [Artifact] for reading. If a implements [FileArtifact],
+// its reader might be used directly, eliminating the roundtrip to vfs.
+// Otherwise, it must cure into a directory containing a single regular file.
 //
-// If err is nil, the caller is responsible for closing the resulting
-// [io.ReadCloser].
+// If err is nil, the caller must close the resulting [io.ReadCloser] and return
+// its error, if any. Failure to read r to EOF may result in a spurious
+// [ChecksumMismatchError], or the underlying implementation may block on Close.
 func (t *TContext) Open(a Artifact) (r io.ReadCloser, err error) {
-	if f, ok := a.(File); ok {
+	if f, ok := a.(FileArtifact); ok {
 		return t.cache.openFile(f)
 	}
 
@@ -274,7 +276,7 @@ func Flood(a Artifact) iter.Seq[Artifact] {
 //
 // TrivialArtifact is unable to cure any other [Artifact] and it cannot access
 // pathnames. This type of [Artifact] is primarily intended for dependency-less
-// artifacts or direct dependencies that only consists of [File].
+// artifacts or direct dependencies that only consists of [FileArtifact].
 type TrivialArtifact interface {
 	// Cure cures the current [Artifact] to the working directory obtained via
 	// [TContext.GetWorkDir].
@@ -309,16 +311,19 @@ type KnownChecksum interface {
 	Checksum() Checksum
 }
 
-// A File refers to an [Artifact] backed by a single file.
-type File interface {
-	// Cure returns the full contents of [File]. If [File] implements
-	// [KnownChecksum], Cure is responsible for validating any data it produces
-	// and must return [ChecksumMismatchError] if validation fails.
+// FileArtifact refers to an [Artifact] backed by a single file.
+type FileArtifact interface {
+	// Cure returns [io.ReadCloser] of the full contents of [FileArtifact]. If
+	// [FileArtifact] implements [KnownChecksum], Cure is responsible for
+	// validating any data it produces and must return [ChecksumMismatchError]
+	// if validation fails. This error is conventionally returned during the
+	// first call to Close, but may be returned during any call to Read before
+	// EOF, or by Cure itself.
 	//
-	// Callers must not modify the returned byte slice.
+	// Callers are responsible for closing the resulting [io.ReadCloser].
 	//
 	// Result must remain identical across multiple invocations.
-	Cure(ctx context.Context) ([]byte, error)
+	Cure(ctx context.Context) (io.ReadCloser, error)
 
 	Artifact
 }
@@ -430,7 +435,7 @@ type Cache struct {
 	// Directory where all [Cache] related files are placed.
 	base *check.Absolute
 
-	// Whether to validate [File.Cure] for a [KnownChecksum] file. This
+	// Whether to validate [FileArtifact.Cure] for a [KnownChecksum] file. This
 	// significantly reduces performance.
 	strict bool
 	// Maximum size of a dependency graph.
@@ -453,6 +458,9 @@ type Cache struct {
 	// Synchronises access to ident and corresponding filesystem entries.
 	identMu sync.RWMutex
 
+	// Buffered I/O free list, must not be accessed directly.
+	bufioPool sync.Pool
+
 	// Unlocks the on-filesystem cache. Must only be called from Close.
 	unlock func()
 	// Synchronises calls to Close.
@@ -573,8 +581,8 @@ func (e *ChecksumMismatchError) Error() string {
 // found and removed from the underlying storage of [Cache].
 type ScrubError struct {
 	// Content-addressed entries not matching their checksum. This can happen
-	// if an incorrect [File] implementation was cured against a non-strict
-	// [Cache].
+	// if an incorrect [FileArtifact] implementation was cured against
+	// a non-strict [Cache].
 	ChecksumMismatches []ChecksumMismatchError
 	// Dangling identifier symlinks. This can happen if the content-addressed
 	// entry was removed while scrubbing due to a checksum mismatch.
@@ -910,10 +918,11 @@ func (c *Cache) finaliseIdent(
 	close(done)
 }
 
-// openFile tries to load [File] from [Cache], and if that fails, obtains it via
-// [File.Cure] instead. Notably, it does not cure [File]. If err is nil, the
-// caller is responsible for closing the resulting [io.ReadCloser].
-func (c *Cache) openFile(f File) (r io.ReadCloser, err error) {
+// openFile tries to load [FileArtifact] from [Cache], and if that fails,
+// obtains it via [FileArtifact.Cure] instead. Notably, it does not cure
+// [FileArtifact] to the filesystem. If err is nil, the caller is responsible
+// for closing the resulting [io.ReadCloser].
+func (c *Cache) openFile(f FileArtifact) (r io.ReadCloser, err error) {
 	if kc, ok := f.(KnownChecksum); ok {
 		c.checksumMu.RLock()
 		r, err = os.Open(c.base.Append(
@@ -943,11 +952,7 @@ func (c *Cache) openFile(f File) (r io.ReadCloser, err error) {
 				}
 			}()
 		}
-		var data []byte
-		if data, err = f.Cure(c.ctx); err != nil {
-			return
-		}
-		r = io.NopCloser(bytes.NewReader(data))
+		return f.Cure(c.ctx)
 	}
 	return
 }
@@ -1217,6 +1222,16 @@ func (c *Cache) exitCure(curesExempt bool) {
 	}
 }
 
+// getWriter is like [bufio.NewWriter] but for bufioPool.
+func (c *Cache) getWriter(w io.Writer) *bufio.Writer {
+	bw := c.bufioPool.Get().(*bufio.Writer)
+	bw.Reset(w)
+	return bw
+}
+
+// putWriter adds bw to bufioPool.
+func (c *Cache) putWriter(bw *bufio.Writer) { c.bufioPool.Put(bw) }
+
 // cure implements Cure without checking the full dependency graph.
 func (c *Cache) cure(a Artifact, curesExempt bool) (
 	pathname *check.Absolute,
@@ -1313,8 +1328,8 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 		}()
 	}
 
-	// cure File outside type switch to skip TContext initialisation
-	if f, ok := a.(File); ok {
+	// cure FileArtifact outside type switch to skip TContext initialisation
+	if f, ok := a.(FileArtifact); ok {
 		if checksumFi != nil {
 			if !checksumFi.Mode().IsRegular() {
 				// unreachable
@@ -1323,67 +1338,96 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 			return
 		}
 
-		var data []byte
+		work := c.base.Append(dirWork, ids)
+		var w *os.File
+		if w, err = os.OpenFile(
+			work.String(),
+			os.O_CREATE|os.O_EXCL|os.O_WRONLY,
+			0400,
+		); err != nil {
+			return
+		}
+		defer func() {
+			closeErr := w.Close()
+			if err == nil {
+				err = closeErr
+			}
+
+			removeErr := os.Remove(work.String())
+			if err == nil && !errors.Is(removeErr, os.ErrNotExist) {
+				err = removeErr
+			}
+		}()
+
+		var r io.ReadCloser
 		if err = c.enterCure(curesExempt); err != nil {
 			return
 		}
-		data, err = f.Cure(c.ctx)
+		r, err = f.Cure(c.ctx)
+		if err == nil {
+			if checksumPathname == nil || c.IsStrict() {
+				h := sha512.New384()
+				hbw := c.getWriter(h)
+				_, err = io.Copy(w, io.TeeReader(r, hbw))
+				flushErr := hbw.Flush()
+				c.putWriter(hbw)
+				if err == nil {
+					err = flushErr
+				}
+
+				if err == nil {
+					buf := c.getIdentBuf()
+					h.Sum(buf[:0])
+
+					if checksumPathname == nil {
+						checksum = unique.Make(Checksum(buf[:]))
+						checksums = Encode(Checksum(buf[:]))
+					} else if c.IsStrict() {
+						if got := Checksum(buf[:]); got != checksum.Value() {
+							err = &ChecksumMismatchError{
+								Got:  got,
+								Want: checksum.Value(),
+							}
+						}
+					}
+
+					c.putIdentBuf(buf)
+
+					if checksumPathname == nil {
+						checksumPathname = c.base.Append(
+							dirChecksum,
+							checksums,
+						)
+					}
+				}
+			} else {
+				_, err = io.Copy(w, r)
+			}
+
+			closeErr := r.Close()
+			if err == nil {
+				err = closeErr
+			}
+		}
 		c.exitCure(curesExempt)
 		if err != nil {
 			return
 		}
 
-		if checksumPathname == nil {
-			h := sha512.New384()
-			h.Write(data)
-			buf := c.getIdentBuf()
-			h.Sum(buf[:0])
-			checksum = unique.Make(Checksum(buf[:]))
-			checksums = Encode(Checksum(buf[:]))
-			c.putIdentBuf(buf)
-			checksumPathname = c.base.Append(
-				dirChecksum,
-				checksums,
-			)
-		} else if c.IsStrict() {
-			h := sha512.New384()
-			h.Write(data)
-			if got := Checksum(h.Sum(nil)); got != checksum.Value() {
-				err = &ChecksumMismatchError{
-					Got:  got,
-					Want: checksum.Value(),
-				}
-				return
-			}
-		}
-
 		c.checksumMu.Lock()
-		var w *os.File
-		w, err = os.OpenFile(
+		if err = os.Rename(
+			work.String(),
 			checksumPathname.String(),
-			os.O_CREATE|os.O_EXCL|os.O_WRONLY,
-			0400,
-		)
-		if err != nil {
+		); err != nil {
 			c.checksumMu.Unlock()
-
-			if errors.Is(err, os.ErrExist) {
-				err = nil
-			}
 			return
 		}
-		_, err = w.Write(data)
-		closeErr := w.Close()
 		timeErr := zeroTimes(checksumPathname.String())
 		c.checksumMu.Unlock()
 
 		if err == nil {
 			err = timeErr
 		}
-		if err == nil {
-			err = closeErr
-		}
-
 		return
 	}
 
@@ -1601,6 +1645,7 @@ func open(
 	}
 	c.ctx, c.cancel = context.WithCancel(ctx)
 	c.identPool.New = func() any { return new(extIdent) }
+	c.bufioPool.New = func() any { return new(bufio.Writer) }
 
 	if lock || !testing.Testing() {
 		if unlock, err := lockedfile.MutexAt(