From 9feac7738f5dbe9f2f32cfa297582d3b520cb049 Mon Sep 17 00:00:00 2001 From: Ophestra Date: Fri, 3 Apr 2026 20:33:59 +0900 Subject: [PATCH] internal/pkg: optionally suppress via assumed checksum This is quite error-prone and causes cache inconsistency similar to the store inconsistency seen on nix when a similar condition happens. Keep this behind a flag in case it is ever beneficial. Signed-off-by: Ophestra --- internal/pkg/dir_test.go | 26 ++++++++++++++++++++ internal/pkg/pkg.go | 34 ++++++++++++++++++++------- internal/pkg/pkg_test.go | 51 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 100 insertions(+), 11 deletions(-) diff --git a/internal/pkg/dir_test.go b/internal/pkg/dir_test.go index cceb89b1..f1a81355 100644 --- a/internal/pkg/dir_test.go +++ b/internal/pkg/dir_test.go @@ -184,6 +184,32 @@ func TestFlatten(t *testing.T) { {Mode: fs.ModeDir | 0700, Path: "work"}, }, pkg.MustDecode("WVpvsVqVKg9Nsh744x57h51AuWUoUR2nnh8Md-EYBQpk6ziyTuUn6PLtF2e0Eu_d"), nil}, + {"sample no assume checksum", fstest.MapFS{ + ".": {Mode: fs.ModeDir | 0700}, + + "checksum": {Mode: fs.ModeDir | 0700}, + "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M": {Mode: fs.ModeDir | 0500}, + "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M/check": {Mode: 0400, Data: []byte{}}, + + "identifier": {Mode: fs.ModeDir | 0700}, + "identifier/_wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, + "identifier/_wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, + + "work": {Mode: fs.ModeDir | 0700}, + }, []pkg.FlatEntry{ + {Mode: fs.ModeDir | 0700, Path: "."}, + + {Mode: fs.ModeDir | 0700, Path: "checksum"}, + {Mode: fs.ModeDir | 0500, Path: "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M"}, + {Mode: 0400, Path: "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M/check", Data: []byte{}}, + + {Mode: fs.ModeDir | 0700, Path: "identifier"}, + {Mode: fs.ModeSymlink | 0777, Path: "identifier/_wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, + {Mode: fs.ModeSymlink | 0777, Path: "identifier/_wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, + + {Mode: fs.ModeDir | 0700, Path: "work"}, + }, pkg.MustDecode("OC290t23aimNo2Rp2pPwan5GI2KRLRdOwYxXQMD9jw0QROgHnNXWodoWdV0hwu2w"), nil}, + {"sample tar step unpack", fstest.MapFS{ ".": {Mode: fs.ModeDir | 0500}, diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go index 0c882bea..ccb7541e 100644 --- a/internal/pkg/pkg.go +++ b/internal/pkg/pkg.go @@ -507,6 +507,20 @@ const ( // CSchedIdle arranges for the [ext.SCHED_IDLE] scheduling priority to be // set for [KindExec] and [KindExecNet] containers. CSchedIdle + + // CAssumeChecksum enables the use of [KnownChecksum] for duplicate function + // call suppression via the on-disk cache. + // + // This may cause incorrect cure outcome if an impossible checksum is + // specified that matches an output already present in the on-disk cache. + // This may be avoided by purposefully specifying a statistically + // unattainable checksum, like the zero value. + // + // While this optimisation might seem appealing, it is almost never + // applicable in real world use. Almost every time this path was taken, it + // was caused by an incorrect checksum accidentally left behind while + // bumping a package. Only enable this if you are really sure you need it. + CAssumeChecksum ) // Cache is a support layer that implementations of [Artifact] can use to store @@ -1041,7 +1055,7 @@ func (c *Cache) finaliseIdent( // [FileArtifact] to the filesystem. If err is nil, the caller is responsible // for closing the resulting [io.ReadCloser]. func (c *Cache) openFile(f FileArtifact) (r io.ReadCloser, err error) { - if kc, ok := f.(KnownChecksum); ok { + if kc, ok := f.(KnownChecksum); c.flags&CAssumeChecksum != 0 && ok { c.checksumMu.RLock() r, err = os.Open(c.base.Append( dirChecksum, @@ -1484,16 +1498,18 @@ func (c *Cache) cure(a Artifact, curesExempt bool) ( checksums, ) - c.checksumMu.RLock() - checksumFi, err = os.Stat(checksumPathname.String()) - c.checksumMu.RUnlock() + if c.flags&CAssumeChecksum != 0 { + c.checksumMu.RLock() + checksumFi, err = os.Stat(checksumPathname.String()) + c.checksumMu.RUnlock() - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return + } + + checksumFi, err = nil, nil } - - checksumFi, err = nil, nil } } diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go index 82f260e7..50e2a584 100644 --- a/internal/pkg/pkg_test.go +++ b/internal/pkg/pkg_test.go @@ -469,7 +469,7 @@ func TestCache(t *testing.T) { }() testCases := []cacheTestCase{ - {"file", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { + {"file", pkg.CValidateKnown | pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { identifier := (pkg.ID)(bytes.Repeat([]byte{ 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f, }, 8)) @@ -626,7 +626,7 @@ func TestCache(t *testing.T) { } }, pkg.MustDecode("St9rlE-mGZ5gXwiv_hzQ_B8bZP-UUvSNmf4nHUZzCMOumb6hKnheZSe0dmnuc4Q2")}, - {"directory", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { + {"directory", pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { id := pkg.MustDecode( "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", ) @@ -873,6 +873,53 @@ func TestCache(t *testing.T) { <-wCureDone }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")}, + {"no assume checksum", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { + makeGarbage := func(work *check.Absolute, wantErr error) error { + if err := os.Mkdir(work.String(), 0700); err != nil { + return err + } + + if err := os.WriteFile(work.Append( + "check", + ).String(), nil, 0400); err != nil { + return err + } + + return wantErr + } + + wantChecksum := pkg.MustDecode("Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M") + + cureMany(t, c, []cureStep{ + {"create", overrideChecksum{wantChecksum, overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{ + kind: pkg.KindTar, + cure: func(t *pkg.TContext) error { + return makeGarbage(t.GetWorkDir(), nil) + }, + }}}, base.Append( + "identifier", + pkg.Encode(pkg.ID{0xff, 0}), + ), wantChecksum, nil}, + + {"reject", overrideChecksum{wantChecksum, overrideIdent{pkg.ID{0xfe, 1}, &stubArtifact{ + kind: pkg.KindTar, + cure: func(t *pkg.TContext) error { + return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xbad)) + }, + }}}, nil, pkg.Checksum{}, stub.UniqueError(0xbad)}, + + {"match", overrideChecksum{wantChecksum, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{ + kind: pkg.KindTar, + cure: func(t *pkg.TContext) error { + return makeGarbage(t.GetWorkDir(), nil) + }, + }}}, base.Append( + "identifier", + pkg.Encode(pkg.ID{0xff, 1}), + ), wantChecksum, nil}, + }) + }, pkg.MustDecode("OC290t23aimNo2Rp2pPwan5GI2KRLRdOwYxXQMD9jw0QROgHnNXWodoWdV0hwu2w")}, + {"scrub", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { cureMany(t, c, []cureStep{ {"bad measured file", newStubFile(