diff --git a/internal/pkg/exec.go b/internal/pkg/exec.go index 8d27d07..f813185 100644 --- a/internal/pkg/exec.go +++ b/internal/pkg/exec.go @@ -2,7 +2,6 @@ package pkg import ( "bufio" - "bytes" "context" "errors" "fmt" @@ -101,8 +100,9 @@ func (a *execNetArtifact) Checksum() Checksum { return a.checksum } func (a *execNetArtifact) Kind() Kind { return KindExecNet } // Params is [Checksum] concatenated with [KindExec] params. -func (a *execNetArtifact) Params() []byte { - return slices.Concat(a.checksum[:], a.execArtifact.Params()) +func (a *execNetArtifact) Params(ctx *IContext) { + ctx.GetHash().Write(a.checksum[:]) + a.execArtifact.Params(ctx) } // Cure cures the [Artifact] in the container described by the caller. The @@ -165,40 +165,40 @@ func NewExec( // Kind returns the hardcoded [Kind] constant. func (a *execArtifact) Kind() Kind { return KindExec } -// Params returns paths, executable pathname and args concatenated together. -func (a *execArtifact) Params() []byte { - var buf bytes.Buffer +// Params writes paths, executable pathname and args. +func (a *execArtifact) Params(ctx *IContext) { + h := ctx.GetHash() + + _0, _1 := []byte{0}, []byte{1} for _, p := range a.paths { if p.W { - buf.WriteByte(1) + h.Write(_1) } else { - buf.WriteByte(0) + h.Write(_0) } if p.P != nil { - buf.WriteString(p.P.String()) + h.Write([]byte(p.P.String())) } else { - buf.WriteString("invalid P\x00") + h.Write([]byte("invalid P\x00")) } - buf.WriteByte(0) + h.Write(_0) for _, d := range p.A { - id := Ident(d) - buf.Write(id[:]) + ctx.WriteIdent(d) } - buf.WriteByte(0) + h.Write(_0) } - buf.WriteByte(0) - buf.WriteString(a.dir.String()) - buf.WriteByte(0) + h.Write(_0) + h.Write([]byte(a.dir.String())) + h.Write(_0) for _, e := range a.env { - buf.WriteString(e) + h.Write([]byte(e)) } - buf.WriteByte(0) - buf.WriteString(a.path.String()) - buf.WriteByte(0) + h.Write(_0) + h.Write([]byte(a.path.String())) + h.Write(_0) for _, arg := range a.args { - buf.WriteString(arg) + h.Write([]byte(arg)) } - return buf.Bytes() } // Dependencies returns a slice of all artifacts collected from caller-supplied diff --git a/internal/pkg/exec_test.go b/internal/pkg/exec_test.go index 97f72a3..558ab39 100644 --- a/internal/pkg/exec_test.go +++ b/internal/pkg/exec_test.go @@ -50,7 +50,7 @@ func TestExec(t *testing.T) { nil, nil, nil, )), - pkg.MustPath("/.hakurei", false, stubArtifact{ + pkg.MustPath("/.hakurei", false, &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), cure: func(t *pkg.TContext) error { @@ -67,7 +67,7 @@ func TestExec(t *testing.T) { check.MustAbs("/opt/bin/testtool"), []string{"testtool"}, - pkg.MustPath("/proc/nonexistent", false, stubArtifact{ + pkg.MustPath("/proc/nonexistent", false, &stubArtifact{ kind: pkg.KindTar, params: []byte("doomed artifact"), cure: func(t *pkg.TContext) error { @@ -124,7 +124,7 @@ func TestExec(t *testing.T) { nil, nil, nil, )), - pkg.MustPath("/.hakurei", false, stubArtifact{ + pkg.MustPath("/.hakurei", false, &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), cure: func(t *pkg.TContext) error { @@ -150,7 +150,7 @@ func TestExec(t *testing.T) { check.MustAbs("/opt/bin/testtool"), []string{"testtool"}, - pkg.MustPath("/", true, stubArtifact{ + pkg.MustPath("/", true, &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), cure: func(t *pkg.TContext) error { @@ -176,13 +176,13 @@ func TestExec(t *testing.T) { check.MustAbs("/work/bin/testtool"), []string{"testtool"}, - pkg.MustPath("/", true, stubArtifact{ + pkg.MustPath("/", true, &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), cure: func(t *pkg.TContext) error { return os.MkdirAll(t.GetWorkDir().String(), 0700) }, - }), pkg.MustPath("/work/", false, stubArtifact{ + }), pkg.MustPath("/work/", false, &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), cure: func(t *pkg.TContext) error { @@ -207,13 +207,13 @@ func TestExec(t *testing.T) { check.MustAbs("/opt/bin/testtool"), []string{"testtool", "layers"}, - pkg.MustPath("/", true, stubArtifact{ + pkg.MustPath("/", true, &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), cure: func(t *pkg.TContext) error { return os.MkdirAll(t.GetWorkDir().String(), 0700) }, - }, stubArtifactF{ + }, &stubArtifactF{ kind: pkg.KindExec, params: []byte("test sample with dependencies"), @@ -222,7 +222,7 @@ func TestExec(t *testing.T) { pkg.ID{0xfe, 0}, nil, nil, nil, - ), stubArtifact{ + ), &stubArtifact{ kind: pkg.KindTar, params: []byte("empty directory"), @@ -255,7 +255,7 @@ func newTesttool() ( testtoolDestroy func(t *testing.T, base *check.Absolute, c *pkg.Cache), ) { // testtoolBin is built during go:generate and is not deterministic - testtool = overrideIdent{pkg.ID{0xfe, 0xff}, stubArtifact{ + testtool = overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { work := t.GetWorkDir() diff --git a/internal/pkg/file.go b/internal/pkg/file.go index 98b8eee..4558eb4 100644 --- a/internal/pkg/file.go +++ b/internal/pkg/file.go @@ -9,7 +9,7 @@ import ( // A fileArtifact is an [Artifact] that cures into data known ahead of time. type fileArtifact []byte -var _ KnownChecksum = fileArtifact{} +var _ KnownChecksum = new(fileArtifact) // fileArtifactNamed embeds fileArtifact alongside a caller-supplied name. type fileArtifactNamed struct { @@ -18,10 +18,11 @@ type fileArtifactNamed struct { name string } -var _ fmt.Stringer = fileArtifactNamed{} +var _ fmt.Stringer = new(fileArtifactNamed) +var _ KnownChecksum = new(fileArtifactNamed) // String returns the caller-supplied reporting name. -func (a fileArtifactNamed) String() string { return a.name } +func (a *fileArtifactNamed) String() string { return a.name } // NewFile returns a [File] that cures into a caller-supplied byte slice. // @@ -29,26 +30,26 @@ func (a fileArtifactNamed) String() string { return a.name } func NewFile(name string, data []byte) File { f := fileArtifact(data) if name != "" { - return fileArtifactNamed{f, name} + return &fileArtifactNamed{f, name} } - return f + return &f } // Kind returns the hardcoded [Kind] constant. -func (a fileArtifact) Kind() Kind { return KindFile } +func (a *fileArtifact) Kind() Kind { return KindFile } -// Params returns the result of Data. -func (a fileArtifact) Params() []byte { return a } +// Params writes the result of Cure. +func (a *fileArtifact) Params(ctx *IContext) { ctx.GetHash().Write(*a) } // Dependencies returns a nil slice. -func (a fileArtifact) Dependencies() []Artifact { return nil } +func (a *fileArtifact) Dependencies() []Artifact { return nil } // Checksum computes and returns the checksum of caller-supplied data. -func (a fileArtifact) Checksum() Checksum { +func (a *fileArtifact) Checksum() Checksum { h := sha512.New384() - h.Write(a) + h.Write(*a) return Checksum(h.Sum(nil)) } // Cure returns the caller-supplied data. -func (a fileArtifact) Cure(context.Context) ([]byte, error) { return a, nil } +func (a *fileArtifact) Cure(context.Context) ([]byte, error) { return *a, nil } diff --git a/internal/pkg/net.go b/internal/pkg/net.go index b8d85b9..c481b52 100644 --- a/internal/pkg/net.go +++ b/internal/pkg/net.go @@ -50,9 +50,11 @@ func NewHTTPGet( // Kind returns the hardcoded [Kind] constant. func (a *httpArtifact) Kind() Kind { return KindHTTPGet } -// Params returns the backing url string. Context is not represented as it does +// Params writes the backing url string. Client is not represented as it does // not affect [Cache.Cure] outcome. -func (a *httpArtifact) Params() []byte { return []byte(a.url) } +func (a *httpArtifact) Params(ctx *IContext) { + ctx.GetHash().Write([]byte(a.url)) +} // Dependencies returns a nil slice. func (a *httpArtifact) Dependencies() []Artifact { return nil } diff --git a/internal/pkg/net_test.go b/internal/pkg/net_test.go index 72460ee..b7f7056 100644 --- a/internal/pkg/net_test.go +++ b/internal/pkg/net_test.go @@ -35,13 +35,10 @@ func TestHTTPGet(t *testing.T) { "file:///testdata", testdataChecksum, ) - wantIdent := pkg.KindHTTPGet.Ident([]byte("file:///testdata")) if got, err := f.Cure(t.Context()); err != nil { t.Fatalf("Cure: error = %v", err) } else if string(got) != testdata { t.Fatalf("Cure: %x, want %x", got, testdata) - } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent)) } // check direct validation @@ -55,8 +52,6 @@ func TestHTTPGet(t *testing.T) { } if _, err := f.Cure(t.Context()); !reflect.DeepEqual(err, wantErrMismatch) { t.Fatalf("Cure: error = %#v, want %#v", err, wantErrMismatch) - } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent)) } // check direct response error @@ -65,12 +60,9 @@ func TestHTTPGet(t *testing.T) { "file:///nonexistent", pkg.Checksum{}, ) - wantIdentNonexistent := pkg.KindHTTPGet.Ident([]byte("file:///nonexistent")) wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound) if _, err := f.Cure(t.Context()); !reflect.DeepEqual(err, wantErrNotFound) { t.Fatalf("Cure: error = %#v, want %#v", err, wantErrNotFound) - } else if gotIdent := pkg.Ident(f); gotIdent != wantIdentNonexistent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdentNonexistent)) } }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")}, @@ -80,10 +72,9 @@ func TestHTTPGet(t *testing.T) { "file:///testdata", testdataChecksum, ) - wantIdent := pkg.KindHTTPGet.Ident([]byte("file:///testdata")) wantPathname := base.Append( "identifier", - pkg.Encode(wantIdent), + "NqVORkT6L9HX6Za7kT2zcibY10qFqBaxEjPiYFrBQX-ZFr3yxCzJxbKOP0zVjeWb", ) if pathname, checksum, err := c.Cure(f); err != nil { t.Fatalf("Cure: error = %v", err) @@ -97,8 +88,6 @@ func TestHTTPGet(t *testing.T) { t.Fatalf("Cure: error = %v", err) } else if string(got) != testdata { t.Fatalf("Cure: %x, want %x", got, testdata) - } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent)) } // check load from cache @@ -111,8 +100,6 @@ func TestHTTPGet(t *testing.T) { t.Fatalf("Cure: error = %v", err) } else if string(got) != testdata { t.Fatalf("Cure: %x, want %x", got, testdata) - } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent)) } // check error passthrough @@ -121,12 +108,9 @@ func TestHTTPGet(t *testing.T) { "file:///nonexistent", pkg.Checksum{}, ) - wantIdentNonexistent := pkg.KindHTTPGet.Ident([]byte("file:///nonexistent")) wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound) if _, _, err := c.Cure(f); !reflect.DeepEqual(err, wantErrNotFound) { t.Fatalf("Pathname: error = %#v, want %#v", err, wantErrNotFound) - } else if gotIdent := pkg.Ident(f); gotIdent != wantIdentNonexistent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdentNonexistent)) } }, pkg.MustDecode("bqtn69RkV5E7V7GhhgCFjcvbxmaqrO8DywamM4Tyjf10F6EJBHjXiIa_tFRtF4iN")}, }) diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go index a9e9d18..d156e75 100644 --- a/internal/pkg/pkg.go +++ b/internal/pkg/pkg.go @@ -9,6 +9,7 @@ import ( "encoding/binary" "errors" "fmt" + "hash" "io" "io/fs" "iter" @@ -20,6 +21,7 @@ import ( "strings" "sync" "syscall" + "unique" "unsafe" "hakurei.app/container/check" @@ -60,6 +62,35 @@ func MustDecode(s string) Checksum { } } +// IContext is passed to [Artifact.Params] and provides identifier information +// and the target [hash.Hash] for writing params into. +// +// Methods of IContext are safe for concurrent use. IContext is valid +// until [Artifact.Params] returns. +type IContext struct { + // Address of underlying [Cache], should be zeroed or made unusable after + // [Artifact.Params] returns and must not be exposed directly. + cache *Cache + // Made available for writing, should be zeroed after [Artifact.Params] + // returns. Internal state must not be inspected. + h hash.Hash +} + +// Unwrap returns the underlying [context.Context]. +func (i *IContext) Unwrap() context.Context { return i.cache.ctx } + +// GetHash returns the underlying [hash.Hash] for writing. Callers must not +// attempt to inspect its internal state. +func (i *IContext) GetHash() hash.Hash { return i.h } + +// WriteIdent writes the identifier of [Artifact] to the underlying [hash.Hash]. +func (i *IContext) WriteIdent(a Artifact) { + buf := i.cache.getIdentBuf() + *(*ID)(buf[wordSize:]) = i.cache.Ident(a).Value() + i.h.Write(buf[wordSize:]) + i.cache.putIdentBuf(buf) +} + // TContext is passed to [TrivialArtifact.Cure] and provides information and // methods required for curing the [TrivialArtifact]. // @@ -154,7 +185,7 @@ type FContext struct { TContext // Cured top-level dependencies looked up by Pathname. - deps map[ID]*check.Absolute + deps map[Artifact]*check.Absolute } // InvalidLookupError is the identifier of non-dependency [Artifact] looked up @@ -171,11 +202,10 @@ var _ error = InvalidLookupError{} // with an [Artifact] not part of the slice returned by [Artifact.Dependencies] // panics. func (f *FContext) Pathname(a Artifact) *check.Absolute { - id := Ident(a) - if p, ok := f.deps[id]; ok { + if p, ok := f.deps[a]; ok { return p } else { - panic(InvalidLookupError(id)) + panic(InvalidLookupError(f.cache.Ident(a).Value())) } } @@ -188,14 +218,13 @@ type Artifact interface { // [Artifact] is allowed to return the same [Kind] value. Kind() Kind - // Params returns opaque bytes that describes [Artifact]. Implementations + // Params writes opaque bytes that describes [Artifact]. Implementations // must guarantee that these values are unique among differing instances - // of the same implementation with the same dependencies. - // - // Callers must not modify the retuned byte slice. + // of the same implementation with the same dependencies. Callers must not + // attempt to interpret these params. // // Result must remain identical across multiple invocations. - Params() []byte + Params(ctx *IContext) // Dependencies returns a slice of [Artifact] that the current instance // depends on to produce its contents. @@ -290,17 +319,9 @@ type File interface { Artifact } -// Ident returns the identifier of an [Artifact]. -func Ident(a Artifact) ID { - if ki, ok := a.(KnownIdent); ok { - return ki.ID() - } - return a.Kind().Ident(a.Params(), a.Dependencies()...) -} - -// reportNameIdent is like reportName but does not recompute [ID]. -func reportNameIdent(a Artifact, id ID) string { - r := Encode(id) +// reportName returns a string describing [Artifact] presented to the user. +func reportName(a Artifact, id unique.Handle[ID]) string { + r := Encode(id.Value()) if s, ok := a.(fmt.Stringer); ok { if name := s.String(); name != "" { r += "-" + name @@ -309,9 +330,6 @@ func reportNameIdent(a Artifact, id ID) string { return r } -// reportName returns a string describing [Artifact] presented to the user. -func reportName(a Artifact) string { return reportNameIdent(a, Ident(a)) } - // Kind corresponds to the concrete type of [Artifact] and is used to create // identifier for an [Artifact] with dependencies. type Kind uint64 @@ -334,31 +352,6 @@ const ( KindCustomOffset = 1 << 31 ) -// Ident returns a deterministic identifier for the supplied params and -// dependencies. The caller is responsible for ensuring params uniquely and -// deterministically describes the current [Artifact]. -func (k Kind) Ident(params []byte, deps ...Artifact) ID { - type extIdent [len(ID{}) + wordSize]byte - identifiers := make([]extIdent, len(deps)) - for i, a := range deps { - id := Ident(a) - copy(identifiers[i][wordSize:], id[:]) - binary.LittleEndian.PutUint64(identifiers[i][:], uint64(a.Kind())) - } - slices.SortFunc(identifiers, func(a, b extIdent) int { - return bytes.Compare(a[:], b[:]) - }) - identifiers = slices.Compact(identifiers) - - h := sha512.New384() - h.Write(binary.LittleEndian.AppendUint64(nil, uint64(k))) - h.Write(params) - for _, e := range identifiers { - h.Write(e[:]) - } - return ID(h.Sum(nil)) -} - const ( // dirIdentifier is the directory name appended to Cache.base for storing // artifacts named after their [ID]. @@ -429,15 +422,20 @@ type Cache struct { // Maximum size of a dependency graph. threshold uintptr + // Artifact to [unique.Handle] of identifier cache. + artifact sync.Map + // Identifier free list, must not be accessed directly. + identPool sync.Pool + // Synchronises access to dirChecksum. checksumMu sync.RWMutex // Identifier to content pair cache. - ident map[ID]Checksum + ident map[unique.Handle[ID]]Checksum // Identifier to error pair for unrecoverably faulted [Artifact]. - identErr map[ID]error + identErr map[unique.Handle[ID]]error // Pending identifiers, accessed through Cure for entries not in ident. - identPending map[ID]<-chan struct{} + identPending map[unique.Handle[ID]]<-chan struct{} // Synchronises access to ident and corresponding filesystem entries. identMu sync.RWMutex } @@ -458,6 +456,89 @@ func (c *Cache) SetStrict(strict bool) { c.strict = strict } // This method is not safe for concurrent use with any other method. func (c *Cache) SetThreshold(threshold uintptr) { c.threshold = threshold } +// extIdent is a [Kind] concatenated with [ID]. +type extIdent [wordSize + len(ID{})]byte + +// getIdentBuf returns the address of an extIdent for Ident. +func (c *Cache) getIdentBuf() *extIdent { return c.identPool.Get().(*extIdent) } + +// putIdentBuf adds buf to identPool. +func (c *Cache) putIdentBuf(buf *extIdent) { c.identPool.Put(buf) } + +// storeIdent adds an [Artifact] to the artifact cache. +func (c *Cache) storeIdent(a Artifact, buf *extIdent) unique.Handle[ID] { + idu := unique.Make(ID(buf[wordSize:])) + c.artifact.Store(a, idu) + return idu +} + +// Ident returns the identifier of an [Artifact]. +func (c *Cache) Ident(a Artifact) unique.Handle[ID] { + buf, idu := c.unsafeIdent(a, false) + if buf != nil { + idu = c.storeIdent(a, buf) + c.putIdentBuf(buf) + } + return idu +} + +// unsafeIdent implements Ident but returns the underlying buffer for a newly +// computed identifier. Callers must return this buffer to identPool. encodeKind +// is only a hint, kind may still be encoded in the buffer. +func (c *Cache) unsafeIdent(a Artifact, encodeKind bool) ( + buf *extIdent, + idu unique.Handle[ID], +) { + if id, ok := c.artifact.Load(a); ok { + idu = id.(unique.Handle[ID]) + return + } + + if ki, ok := a.(KnownIdent); ok { + buf = c.getIdentBuf() + if encodeKind { + binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) + } + *(*ID)(buf[wordSize:]) = ki.ID() + return + } + + deps := a.Dependencies() + idents := make([]*extIdent, len(deps)) + for i, d := range deps { + dbuf, did := c.unsafeIdent(d, true) + if dbuf == nil { + dbuf = c.getIdentBuf() + binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind())) + *(*ID)(dbuf[wordSize:]) = did.Value() + } else { + c.storeIdent(d, dbuf) + } + defer c.putIdentBuf(dbuf) + idents[i] = dbuf + } + slices.SortFunc(idents, func(a, b *extIdent) int { + return bytes.Compare(a[:], b[:]) + }) + idents = slices.CompactFunc(idents, func(a, b *extIdent) bool { + return *a == *b + }) + + buf = c.getIdentBuf() + h := sha512.New384() + binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) + h.Write(buf[:wordSize]) + i := IContext{c, h} + a.Params(&i) + i.cache, i.h = nil, nil + for _, dn := range idents { + h.Write(dn[:]) + } + + h.Sum(buf[wordSize:wordSize]) + return +} + // A ChecksumMismatchError describes an [Artifact] with unexpected content. type ChecksumMismatchError struct { // Actual and expected checksums. @@ -535,8 +616,8 @@ func (c *Cache) Scrub() error { c.checksumMu.Lock() defer c.checksumMu.Unlock() - c.ident = make(map[ID]Checksum) - c.identErr = make(map[ID]error) + c.ident = make(map[unique.Handle[ID]]Checksum) + c.identErr = make(map[unique.Handle[ID]]error) var se ScrubError @@ -687,7 +768,7 @@ func (c *Cache) Scrub() error { // loadOrStoreIdent attempts to load a cached [Artifact] by its identifier or // wait for a pending [Artifact] to cure. If neither is possible, the current // identifier is stored in identPending and a non-nil channel is returned. -func (c *Cache) loadOrStoreIdent(id *ID) ( +func (c *Cache) loadOrStoreIdent(id unique.Handle[ID]) ( done chan<- struct{}, checksum Checksum, err error, @@ -695,29 +776,29 @@ func (c *Cache) loadOrStoreIdent(id *ID) ( var ok bool c.identMu.Lock() - if checksum, ok = c.ident[*id]; ok { + if checksum, ok = c.ident[id]; ok { c.identMu.Unlock() return } - if err, ok = c.identErr[*id]; ok { + if err, ok = c.identErr[id]; ok { c.identMu.Unlock() return } var notify <-chan struct{} - if notify, ok = c.identPending[*id]; ok { + if notify, ok = c.identPending[id]; ok { c.identMu.Unlock() <-notify c.identMu.RLock() - if checksum, ok = c.ident[*id]; !ok { - err = c.identErr[*id] + if checksum, ok = c.ident[id]; !ok { + err = c.identErr[id] } c.identMu.RUnlock() return } d := make(chan struct{}) - c.identPending[*id] = d + c.identPending[id] = d c.identMu.Unlock() done = d return @@ -727,17 +808,17 @@ func (c *Cache) loadOrStoreIdent(id *ID) ( // previously submitted to identPending. func (c *Cache) finaliseIdent( done chan<- struct{}, - id *ID, + id unique.Handle[ID], checksum *Checksum, err error, ) { c.identMu.Lock() if err != nil { - c.identErr[*id] = err + c.identErr[id] = err } else { - c.ident[*id] = *checksum + c.ident[id] = *checksum } - delete(c.identPending, *id) + delete(c.identPending, id) c.identMu.Unlock() close(done) @@ -758,7 +839,7 @@ func (c *Cache) openFile(f File) (r io.ReadCloser, err error) { c.identMu.RLock() r, err = os.Open(c.base.Append( dirIdentifier, - Encode(Ident(f)), + Encode(c.Ident(f).Value()), ).String()) c.identMu.RUnlock() } @@ -768,7 +849,7 @@ func (c *Cache) openFile(f File) (r io.ReadCloser, err error) { return } if c.msg.IsVerbose() { - rn := reportName(f) + rn := reportName(f, c.Ident(f)) c.msg.Verbosef("curing %s to memory...", rn) defer func() { if err == nil { @@ -955,8 +1036,8 @@ func (c *Cache) cure(a Artifact) ( checksum Checksum, err error, ) { - id := Ident(a) - ids := Encode(id) + id := c.Ident(a) + ids := Encode(id.Value()) pathname = c.base.Append( dirIdentifier, ids, @@ -969,11 +1050,11 @@ func (c *Cache) cure(a Artifact) ( }() var done chan<- struct{} - done, checksum, err = c.loadOrStoreIdent(&id) + done, checksum, err = c.loadOrStoreIdent(id) if done == nil { return } else { - defer func() { c.finaliseIdent(done, &id, &checksum, err) }() + defer func() { c.finaliseIdent(done, id, &checksum, err) }() } _, err = os.Lstat(pathname.String()) @@ -1026,7 +1107,7 @@ func (c *Cache) cure(a Artifact) ( } if c.msg.IsVerbose() { - rn := reportNameIdent(a, id) + rn := reportName(a, id) c.msg.Verbosef("curing %s...", rn) defer func() { if err != nil { @@ -1126,7 +1207,7 @@ func (c *Cache) cure(a Artifact) ( case FloodArtifact: deps := a.Dependencies() - f := FContext{t, make(map[ID]*check.Absolute, len(deps))} + f := FContext{t, make(map[Artifact]*check.Absolute, len(deps))} var wg sync.WaitGroup wg.Add(len(deps)) @@ -1155,7 +1236,7 @@ func (c *Cache) cure(a Artifact) ( return } for i, p := range res { - f.deps[Ident(deps[i])] = p + f.deps[deps[i]] = p } defer f.destroy(&err) @@ -1165,7 +1246,7 @@ func (c *Cache) cure(a Artifact) ( break default: - err = InvalidArtifactError(id) + err = InvalidArtifactError(id.Value()) return } t.cache = nil @@ -1285,13 +1366,14 @@ func New( msg: msg, base: base, - ident: make(map[ID]Checksum), - identErr: make(map[ID]error), - identPending: make(map[ID]<-chan struct{}), + ident: make(map[unique.Handle[ID]]Checksum), + identErr: make(map[unique.Handle[ID]]error), + identPending: make(map[unique.Handle[ID]]<-chan struct{}), } c.ctx, c.cancel = context.WithCancel(ctx) cureDep := make(chan *pendingArtifactDep, cures) c.cureDep = cureDep + c.identPool.New = func() any { return new(extIdent) } if cures < 1 { cures = runtime.NumCPU() diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go index f5e6dc5..bb48dfa 100644 --- a/internal/pkg/pkg_test.go +++ b/internal/pkg/pkg_test.go @@ -6,7 +6,6 @@ import ( "context" "crypto/sha512" "encoding/base64" - "encoding/binary" "errors" "fmt" "io" @@ -18,6 +17,7 @@ import ( "reflect" "syscall" "testing" + "unique" "unsafe" "hakurei.app/container" @@ -82,10 +82,10 @@ type stubArtifact struct { cure func(t *pkg.TContext) error } -func (a stubArtifact) Kind() pkg.Kind { return a.kind } -func (a stubArtifact) Params() []byte { return a.params } -func (a stubArtifact) Dependencies() []pkg.Artifact { return a.deps } -func (a stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) } +func (a *stubArtifact) Kind() pkg.Kind { return a.kind } +func (a *stubArtifact) Params(ctx *pkg.IContext) { ctx.GetHash().Write(a.params) } +func (a *stubArtifact) Dependencies() []pkg.Artifact { return a.deps } +func (a *stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) } // A stubArtifactF implements [FloodArtifact] with hardcoded behaviour. type stubArtifactF struct { @@ -96,10 +96,10 @@ type stubArtifactF struct { cure func(f *pkg.FContext) error } -func (a stubArtifactF) Kind() pkg.Kind { return a.kind } -func (a stubArtifactF) Params() []byte { return a.params } -func (a stubArtifactF) Dependencies() []pkg.Artifact { return a.deps } -func (a stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) } +func (a *stubArtifactF) Kind() pkg.Kind { return a.kind } +func (a *stubArtifactF) Params(ctx *pkg.IContext) { ctx.GetHash().Write(a.params) } +func (a *stubArtifactF) Dependencies() []pkg.Artifact { return a.deps } +func (a *stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) } // A stubFile implements [File] with hardcoded behaviour. type stubFile struct { @@ -109,7 +109,7 @@ type stubFile struct { stubArtifact } -func (a stubFile) Cure(context.Context) ([]byte, error) { return a.data, a.err } +func (a *stubFile) Cure(context.Context) ([]byte, error) { return a.data, a.err } // newStubFile returns an implementation of [pkg.File] with hardcoded behaviour. func newStubFile( @@ -119,7 +119,7 @@ func newStubFile( data []byte, err error, ) pkg.File { - f := overrideIdentFile{id, stubFile{data, err, stubArtifact{ + f := overrideIdentFile{id, &stubFile{data, err, stubArtifact{ kind, nil, nil, @@ -193,27 +193,38 @@ func TestIdent(t *testing.T) { testCases := []struct { name string a pkg.Artifact - want pkg.ID + want unique.Handle[pkg.ID] }{ - {"tar", stubArtifact{ + {"tar", &stubArtifact{ pkg.KindTar, []byte{pkg.TarGzip, 0, 0, 0, 0, 0, 0, 0}, []pkg.Artifact{ - overrideIdent{pkg.ID{}, stubArtifact{}}, + overrideIdent{pkg.ID{}, new(stubArtifact)}, }, nil, - }, pkg.MustDecode( + }, unique.Make[pkg.ID](pkg.MustDecode( "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", - )}, + ))}, } + + msg := message.New(log.New(os.Stderr, "ident: ", 0)) + msg.SwapVerbose(true) + var cache *pkg.Cache + if a, err := check.NewAbs(t.TempDir()); err != nil { + t.Fatal(err) + } else if cache, err = pkg.New(t.Context(), msg, 0, a); err != nil { + t.Fatal(err) + } + t.Cleanup(cache.Close) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - if got := pkg.Ident(tc.a); got != tc.want { + if got := cache.Ident(tc.a); got != tc.want { t.Errorf("Ident: %s, want %s", - pkg.Encode(got), - pkg.Encode(tc.want), + pkg.Encode(got.Value()), + pkg.Encode(tc.want.Value()), ) } }) @@ -438,7 +449,7 @@ func TestCache(t *testing.T) { 0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17, }, nil}, - {"incomplete implementation", struct{ pkg.Artifact }{stubArtifact{ + {"incomplete implementation", struct{ pkg.Artifact }{&stubArtifact{ kind: pkg.KindExec, params: []byte("artifact overridden to be incomplete"), }}, nil, pkg.Checksum{}, pkg.InvalidArtifactError(pkg.MustDecode( @@ -459,7 +470,7 @@ func TestCache(t *testing.T) { nil, nil, ), nil, pkg.Checksum{}, stub.UniqueError(0xcafe)}, - {"cache hit bad type", overrideChecksum{testdataChecksum, overrideIdent{pkg.ID{0xff, 2}, stubArtifact{ + {"cache hit bad type", overrideChecksum{testdataChecksum, overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{ kind: pkg.KindTar, }}}, nil, pkg.Checksum{}, pkg.InvalidFileModeError( 0400, @@ -493,7 +504,7 @@ func TestCache(t *testing.T) { // cure after close c.Close() - if _, _, err = c.Cure(stubArtifactF{ + if _, _, err = c.Cure(&stubArtifactF{ kind: pkg.KindExec, params: []byte("unreachable artifact cured after cancel"), deps: []pkg.Artifact{pkg.NewFile("", []byte("unreachable dependency"))}, @@ -504,9 +515,8 @@ func TestCache(t *testing.T) { }, pkg.MustDecode("St9rlE-mGZ5gXwiv_hzQ_B8bZP-UUvSNmf4nHUZzCMOumb6hKnheZSe0dmnuc4Q2")}, {"directory", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - id := pkg.KindTar.Ident( - binary.LittleEndian.AppendUint64(nil, pkg.TarGzip), - overrideIdent{testdataChecksum, stubArtifact{}}, + id := pkg.MustDecode( + "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", ) makeSample := func(t *pkg.TContext) error { work := t.GetWorkDir() @@ -545,9 +555,8 @@ func TestCache(t *testing.T) { pkg.Encode(id), ) - id0 := pkg.KindTar.Ident( - binary.LittleEndian.AppendUint64(nil, pkg.TarGzip), - overrideIdent{pkg.ID{}, stubArtifact{}}, + id0 := pkg.MustDecode( + "Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa", ) wantPathname0 := base.Append( "identifier", @@ -594,28 +603,28 @@ func TestCache(t *testing.T) { } cureMany(t, c, []cureStep{ - {"initial directory", overrideChecksum{wantChecksum, overrideIdent{id, stubArtifact{ + {"initial directory", overrideChecksum{wantChecksum, overrideIdent{id, &stubArtifact{ kind: pkg.KindTar, cure: makeSample, }}}, wantPathname, wantChecksum, nil}, - {"identical identifier", overrideChecksum{wantChecksum, overrideIdent{id, stubArtifact{ + {"identical identifier", overrideChecksum{wantChecksum, overrideIdent{id, &stubArtifact{ kind: pkg.KindTar, }}}, wantPathname, wantChecksum, nil}, - {"identical checksum", overrideIdent{id0, stubArtifact{ + {"identical checksum", overrideIdent{id0, &stubArtifact{ kind: pkg.KindTar, cure: makeSample, }}, wantPathname0, wantChecksum, nil}, - {"cure fault", overrideIdent{pkg.ID{0xff, 0}, stubArtifact{ + {"cure fault", overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe)) }, }}, nil, pkg.Checksum{}, stub.UniqueError(0xcafe)}, - {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, stubArtifact{ + {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { return makeGarbage(t.GetWorkDir(), nil) @@ -635,7 +644,7 @@ func TestCache(t *testing.T) { fs.ModeDir | 0500, )}, - {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, stubArtifact{ + {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { r, err := t.Open(overrideChecksumFile{checksum: wantChecksum}) @@ -654,21 +663,21 @@ func TestCache(t *testing.T) { Err: syscall.EISDIR, }}, - {"no output", overrideIdent{pkg.ID{0xff, 4}, stubArtifact{ + {"no output", overrideIdent{pkg.ID{0xff, 4}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { return nil }, }}, nil, pkg.Checksum{}, pkg.NoOutputError{}}, - {"file output", overrideIdent{pkg.ID{0xff, 5}, stubArtifact{ + {"file output", overrideIdent{pkg.ID{0xff, 5}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400) }, }}, nil, pkg.Checksum{}, errors.New("non-file artifact produced regular file")}, - {"symlink output", overrideIdent{pkg.ID{0xff, 6}, stubArtifact{ + {"symlink output", overrideIdent{pkg.ID{0xff, 6}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { return os.Symlink( @@ -688,7 +697,7 @@ func TestCache(t *testing.T) { wantErr := stub.UniqueError(0xcafe) n, ready := make(chan struct{}), make(chan struct{}) go func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{ + if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { close(ready) @@ -703,7 +712,7 @@ func TestCache(t *testing.T) { <-ready wCureDone := make(chan struct{}) go func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{ + if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{ kind: pkg.KindTar, }}); !reflect.DeepEqual(err, wantErr) { panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr)) @@ -720,7 +729,7 @@ func TestCache(t *testing.T) { nil, stub.UniqueError(0xbad), ), nil, pkg.Checksum{}, stub.UniqueError(0xbad)}, - {"file output", overrideIdent{pkg.ID{0xff, 2}, stubArtifact{ + {"file output", overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{ kind: pkg.KindTar, cure: func(t *pkg.TContext) error { return os.WriteFile( @@ -745,8 +754,8 @@ func TestCache(t *testing.T) { identPending := reflect.NewAt( identPendingVal.Type(), unsafe.Pointer(identPendingVal.UnsafeAddr()), - ).Elem().Interface().(map[pkg.ID]<-chan struct{}) - notify := identPending[pkg.ID{0xff}] + ).Elem().Interface().(map[unique.Handle[pkg.ID]]<-chan struct{}) + notify := identPending[unique.Make(pkg.ID{0xff})] go close(n) <-notify <-wCureDone diff --git a/internal/pkg/tar.go b/internal/pkg/tar.go index eda4140..faad485 100644 --- a/internal/pkg/tar.go +++ b/internal/pkg/tar.go @@ -70,9 +70,9 @@ func NewHTTPGetTar( // Kind returns the hardcoded [Kind] constant. func (a *tarArtifact) Kind() Kind { return KindTar } -// Params returns compression encoded in little endian. -func (a *tarArtifact) Params() []byte { - return binary.LittleEndian.AppendUint64(nil, a.compression) +// Params writes compression encoded in little endian. +func (a *tarArtifact) Params(ctx *IContext) { + ctx.GetHash().Write(binary.LittleEndian.AppendUint64(nil, a.compression)) } // Dependencies returns a slice containing the backing file. diff --git a/internal/pkg/tar_test.go b/internal/pkg/tar_test.go index f468b62..d9635fb 100644 --- a/internal/pkg/tar_test.go +++ b/internal/pkg/tar_test.go @@ -101,8 +101,11 @@ func checkTarHTTP( h.Write([]byte{byte(pkg.KindTar), 0, 0, 0, 0, 0, 0, 0}) h.Write([]byte{pkg.TarGzip, 0, 0, 0, 0, 0, 0, 0}) h.Write([]byte{byte(pkg.KindHTTPGet), 0, 0, 0, 0, 0, 0, 0}) - httpIdent := pkg.KindHTTPGet.Ident([]byte("file:///testdata")) - h.Write(httpIdent[:]) + + h0 := sha512.New384() + h0.Write([]byte{byte(pkg.KindHTTPGet), 0, 0, 0, 0, 0, 0, 0}) + h0.Write([]byte("file:///testdata")) + h.Write(h0.Sum(nil)) return pkg.ID(h.Sum(nil)) }() @@ -113,10 +116,6 @@ func checkTarHTTP( pkg.TarGzip, ) - if id := pkg.Ident(a); id != wantIdent { - t.Fatalf("Ident: %s, want %s", pkg.Encode(id), pkg.Encode(wantIdent)) - } - tarDir := stubArtifact{ kind: pkg.KindExec, params: []byte("directory containing a single regular file"), @@ -164,9 +163,9 @@ func checkTarHTTP( }, } // destroy these to avoid including it in flatten test case - defer newDestroyArtifactFunc(tarDir)(t, base, c) - defer newDestroyArtifactFunc(tarDirMulti)(t, base, c) - defer newDestroyArtifactFunc(tarDirType)(t, base, c) + defer newDestroyArtifactFunc(&tarDir)(t, base, c) + defer newDestroyArtifactFunc(&tarDirMulti)(t, base, c) + defer newDestroyArtifactFunc(&tarDirType)(t, base, c) cureMany(t, c, []cureStep{ {"file", a, base.Append( @@ -175,25 +174,25 @@ func checkTarHTTP( ), wantChecksum, nil}, {"directory", pkg.NewTar( - tarDir, + &tarDir, pkg.TarGzip, ), ignorePathname, wantChecksum, nil}, {"multiple entries", pkg.NewTar( - tarDirMulti, + &tarDirMulti, pkg.TarGzip, ), nil, pkg.Checksum{}, errors.New( "input directory does not contain a single regular file", )}, {"bad type", pkg.NewTar( - tarDirType, + &tarDirType, pkg.TarGzip, ), nil, pkg.Checksum{}, errors.New( "input directory does not contain a single regular file", )}, - {"error passthrough", pkg.NewTar(stubArtifact{ + {"error passthrough", pkg.NewTar(&stubArtifact{ kind: pkg.KindExec, params: []byte("doomed artifact"), cure: func(t *pkg.TContext) error {