internal/pkg: implement caching for directories
All checks were successful
Test / Hpkg (push) Successful in 4m29s
Test / Sandbox (race detector) (push) Successful in 4m50s
Test / Hakurei (race detector) (push) Successful in 5m49s
Test / Flake checks (push) Successful in 1m43s
Test / Create distribution (push) Successful in 45s
Test / Sandbox (push) Successful in 2m30s
Test / Hakurei (push) Successful in 3m38s
Test / ShareFS (push) Successful in 3m40s

This works on any directories and should be robust against any bad state the artifact curing process might have failed at.

Signed-off-by: Ophestra <cat@gensokyo.uk>
This commit is contained in:
2026-01-03 22:49:01 +09:00
parent 863d3dcf9f
commit 40081e7a06
3 changed files with 342 additions and 24 deletions

View File

@@ -20,68 +20,136 @@ func TestFlatten(t *testing.T) {
sum pkg.Checksum
}{
{"empty", fstest.MapFS{
".": {Mode: 020000000700},
"checksum": {Mode: 020000000700},
"identifier": {Mode: 020000000700},
"work": {Mode: 020000000700},
".": {Mode: fs.ModeDir | 0700},
"checksum": {Mode: fs.ModeDir | 0700},
"identifier": {Mode: fs.ModeDir | 0700},
"work": {Mode: fs.ModeDir | 0700},
}, []pkg.FlatEntry{
{Mode: 020000000700, Path: "."},
{Mode: 020000000700, Path: "checksum"},
{Mode: 020000000700, Path: "identifier"},
{Mode: 020000000700, Path: "work"},
{Mode: fs.ModeDir | 0700, Path: "."},
{Mode: fs.ModeDir | 0700, Path: "checksum"},
{Mode: fs.ModeDir | 0700, Path: "identifier"},
{Mode: fs.ModeDir | 0700, Path: "work"},
}, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")},
{"sample cache file", fstest.MapFS{
".": {Mode: 020000000700},
".": {Mode: fs.ModeDir | 0700},
"checksum": {Mode: 020000000700},
"checksum": {Mode: fs.ModeDir | 0700},
"checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}},
"checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
"identifier": {Mode: 020000000700},
"identifier": {Mode: fs.ModeDir | 0700},
"identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}},
"identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
"identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
"identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
"work": {Mode: 020000000700},
"work": {Mode: fs.ModeDir | 0700},
}, []pkg.FlatEntry{
{Mode: 020000000700, Path: "."},
{Mode: fs.ModeDir | 0700, Path: "."},
{Mode: 020000000700, Path: "checksum"},
{Mode: fs.ModeDir | 0700, Path: "checksum"},
{Mode: 0400, Path: "checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
{Mode: 0400, Path: "checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", Data: []byte{0}},
{Mode: 020000000700, Path: "identifier"},
{Mode: fs.ModeDir | 0700, Path: "identifier"},
{Mode: 0400, Path: "identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
{Mode: 0400, Path: "identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
{Mode: 0400, Path: "identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
{Mode: 0400, Path: "identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", Data: []byte{0}},
{Mode: 020000000700, Path: "work"},
{Mode: fs.ModeDir | 0700, Path: "work"},
}, pkg.MustDecode("G4u4W77C3u46oSAzwPTERKbS9h76iIvcd7Zl8p8Y6hTMb4_QGpH0Glg_DIJg-Usa")},
{"sample load or store", fstest.MapFS{
".": {Mode: 020000000700},
".": {Mode: fs.ModeDir | 0700},
"checksum": {Mode: 020000000700},
"checksum": {Mode: fs.ModeDir | 0700},
"checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")},
"identifier": {Mode: 020000000700},
"identifier": {Mode: fs.ModeDir | 0700},
"identifier/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")},
"work": {Mode: 020000000700},
"work": {Mode: fs.ModeDir | 0700},
}, []pkg.FlatEntry{
{Mode: 020000000700, Path: "."},
{Mode: fs.ModeDir | 0700, Path: "."},
{Mode: 020000000700, Path: "checksum"},
{Mode: fs.ModeDir | 0700, Path: "checksum"},
{Mode: 0400, Path: "checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU", Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")},
{Mode: 020000000700, Path: "identifier"},
{Mode: fs.ModeDir | 0700, Path: "identifier"},
{Mode: 0400, Path: "identifier/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU", Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")},
{Mode: 020000000700, Path: "work"},
{Mode: fs.ModeDir | 0700, Path: "work"},
}, pkg.MustDecode("bAJdeuI2cITlff5uIDbMKIiaW6zB-_XGinOMOMjNmltL3YAanXLiV64_jsXeQMXn")},
{"sample directory step simple", fstest.MapFS{
".": {Mode: fs.ModeDir | 0700},
"check": {Mode: 0400, Data: []byte{0, 0}},
"lib": {Mode: fs.ModeDir | 0700},
"lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
"lib/pkgconfig": {Mode: fs.ModeDir | 0700},
}, []pkg.FlatEntry{
{Mode: fs.ModeDir | 0700, Path: "."},
{Mode: 0400, Path: "check", Data: []byte{0x0, 0x0}},
{Mode: fs.ModeDir | 0700, Path: "lib"},
{Mode: fs.ModeSymlink | 0777, Path: "lib/libedac.so", Data: []byte("/proc/nonexistent/libedac.so")},
{Mode: fs.ModeDir | 0700, Path: "lib/pkgconfig"},
}, pkg.MustDecode("1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
{"sample directory step garbage", fstest.MapFS{
".": {Mode: fs.ModeDir | 0700},
"lib": {Mode: fs.ModeDir | 0500},
"lib/check": {Mode: 0400, Data: []byte{}},
"lib/pkgconfig": {Mode: fs.ModeDir | 0500},
}, []pkg.FlatEntry{
{Mode: fs.ModeDir | 0700, Path: "."},
{Mode: fs.ModeDir | 0500, Path: "lib"},
{Mode: 0400, Path: "lib/check", Data: []byte{}},
{Mode: fs.ModeDir | 0500, Path: "lib/pkgconfig"},
}, pkg.MustDecode("GbjlYMcHQANdfwL6qNGopBF99IscPTvCy95HSH1_kIF3eKjFDSLP0_iUUT0z8hiw")},
{"sample directory", fstest.MapFS{
".": {Mode: fs.ModeDir | 0700},
"checksum": {Mode: fs.ModeDir | 0700},
"checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0700},
"checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}},
"checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0700},
"checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0700},
"checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: 01000000777, Data: []byte("/proc/nonexistent/libedac.so")},
"identifier": {Mode: fs.ModeDir | 0700},
"identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: 01000000777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
"identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: 01000000777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
"work": {Mode: fs.ModeDir | 0700},
}, []pkg.FlatEntry{
{Mode: fs.ModeDir | 0700, Path: "."},
{Mode: fs.ModeDir | 0700, Path: "checksum"},
{Mode: fs.ModeDir | 0700, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP"},
{Mode: 0400, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check", Data: []byte{0, 0}},
{Mode: fs.ModeDir | 0700, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib"},
{Mode: fs.ModeSymlink | 0777, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so", Data: []byte("/proc/nonexistent/libedac.so")},
{Mode: fs.ModeDir | 0700, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig"},
{Mode: fs.ModeDir | 0700, Path: "identifier"},
{Mode: fs.ModeSymlink | 0777, Path: "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
{Mode: fs.ModeSymlink | 0777, Path: "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa", Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
{Mode: fs.ModeDir | 0700, Path: "work"},
}, pkg.MustDecode("N7dntFYbOq9V4iC-rjAQ-By6ofPIQVZkA8V0r0G07M_sdB7Zh42Ttrspsc38ioYa")},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {

View File

@@ -8,7 +8,10 @@ import (
"encoding/binary"
"errors"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"slices"
"sync"
@@ -190,6 +193,100 @@ func (c *Cache) pathnameIdent(id *ID) *check.Absolute {
)
}
// Store looks up an identifier, and if it is not present, calls makeArtifact
// with a private working directory and stores its result instead. An optional
// checksum can be passed via the result buffer which is used to validate the
// produced directory.
func (c *Cache) Store(
id ID,
makeArtifact func(work *check.Absolute) error,
buf *Checksum,
validate bool,
) (
pathname *check.Absolute,
store bool,
err error,
) {
pathname = c.pathnameIdent(&id)
c.mu.Lock()
defer c.mu.Unlock()
_, err = os.Lstat(pathname.String())
if err == nil || !errors.Is(err, os.ErrNotExist) {
return
}
store = true
var (
workPathname *check.Absolute
workPathnameRaw string
)
if workPathnameRaw, err = os.MkdirTemp(
c.base.Append(dirWork).String(),
path.Base(pathname.String()+".*"),
); err != nil {
return
} else if workPathname, err = check.NewAbs(workPathnameRaw); err != nil {
return
}
defer func() {
if err != nil {
chmodErr := filepath.WalkDir(workPathname.String(), func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return os.Chmod(path, 0700)
}
return nil
})
removeErr := os.RemoveAll(workPathname.String())
if chmodErr != nil || removeErr != nil {
err = errors.Join(err, chmodErr, removeErr)
} else if errors.Is(err, os.ErrExist) {
// two artifacts may be backed by the same file
err = nil
}
}
}()
if err = os.Chmod(workPathname.String(), 0700); err != nil {
return
}
if err = makeArtifact(workPathname); err != nil {
return
}
var checksum Checksum
if checksum, err = HashDir(workPathname); err != nil {
return
}
if validate {
if checksum != *buf {
err = &ChecksumMismatchError{checksum, *buf}
return
}
} else {
*buf = checksum
}
checksumPathname := c.pathname(&checksum)
if err = os.Rename(
workPathname.String(),
checksumPathname.String(),
); err != nil {
if !errors.Is(err, os.ErrExist) {
return
}
}
if linkErr := os.Symlink(
"../"+dirChecksum+"/"+path.Base(checksumPathname.String()),
pathname.String(),
); linkErr != nil {
err = linkErr
}
return
}
// storeFile stores the contents of a [File]. An optional checksum can be
// passed via the result buffer which is used to validate the submitted data.
//

View File

@@ -3,6 +3,7 @@ package pkg_test
import (
"bytes"
"crypto/sha512"
"encoding/binary"
"io/fs"
"net/http"
"os"
@@ -303,6 +304,158 @@ func TestCache(t *testing.T) {
t.Fatalf("LoadFile: error = %#v, want %#v", err, wantErrNonexistentZero)
}
}, pkg.MustDecode("G4u4W77C3u46oSAzwPTERKbS9h76iIvcd7Zl8p8Y6hTMb4_QGpH0Glg_DIJg-Usa")},
{"directory", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
id := pkg.KindTar.Ident(
binary.LittleEndian.AppendUint64(nil, 1),
stubArtifact{pkg.KindHTTP, testdataChecksum},
)
makeSample := func(work *check.Absolute) error {
if err := os.WriteFile(
work.Append("check").String(),
[]byte{0, 0},
0400,
); err != nil {
return err
}
if err := os.MkdirAll(work.Append(
"lib",
"pkgconfig",
).String(), 0700); err != nil {
return err
}
return os.Symlink(
"/proc/nonexistent/libedac.so",
work.Append(
"lib",
"libedac.so",
).String(),
)
}
wantChecksum := pkg.MustDecode(
"1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP",
)
wantPathname := base.Append(
"identifier",
pkg.Encode(id),
)
if pathname, store, err := c.Store(
id,
makeSample,
&wantChecksum,
true,
); err != nil {
t.Fatalf("Store: error = %v", err)
} else if !store {
t.Fatal("Store did not store nonpresent entry")
} else if !pathname.Is(wantPathname) {
t.Fatalf("Store: pathname = %q, want %q", pathname, wantPathname)
}
// check lookup
if pathname, store, err := c.Store(
id,
nil,
&wantChecksum,
true,
); err != nil {
t.Fatalf("Store: error = %v", err)
} else if store {
t.Fatal("Store stored over present entry")
} else if !pathname.Is(wantPathname) {
t.Fatalf("Store: pathname = %q, want %q", pathname, wantPathname)
}
// check exist
id0 := pkg.KindTar.Ident(
binary.LittleEndian.AppendUint64(nil, 1),
stubArtifact{pkg.KindHTTP, pkg.ID{}},
)
wantPathname0 := base.Append(
"identifier",
pkg.Encode(id0),
)
if pathname, store, err := c.Store(
id0,
makeSample,
&wantChecksum,
true,
); err != nil {
t.Fatalf("Store: error = %v", err)
} else if !store {
t.Fatal("Store did not store nonpresent entry")
} else if !pathname.Is(wantPathname0) {
t.Fatalf("Store: pathname = %q, want %q", pathname, wantPathname0)
}
var wantErrMakeGarbage error
makeGarbage := func(work *check.Absolute) error {
mode := fs.FileMode(0)
if wantErrMakeGarbage == nil {
mode = 0500
}
if err := os.MkdirAll(work.Append(
"lib",
"pkgconfig",
).String(), 0700); err != nil {
return err
}
if err := os.WriteFile(work.Append(
"lib",
"check",
).String(), nil, 0400&mode); err != nil {
return err
}
if err := os.Chmod(work.Append(
"lib",
"pkgconfig",
).String(), 0500&mode); err != nil {
return err
}
if err := os.Chmod(work.Append(
"lib",
).String(), 0500&mode); err != nil {
return err
}
return wantErrMakeGarbage
}
// check makeArtifact fault
wantErrMakeGarbage = stub.UniqueError(0xcafe)
if _, store, err := c.Store(
pkg.ID{},
makeGarbage,
nil,
false,
); !reflect.DeepEqual(err, wantErrMakeGarbage) {
t.Fatalf("Store: error = %#v, want %#v", err, wantErrMakeGarbage)
} else if !store {
t.Fatal("Store did not store nonpresent entry")
}
// checksum mismatch
wantErrMakeGarbage = nil
wantErrMismatch := &pkg.ChecksumMismatchError{
Got: pkg.MustDecode("GbjlYMcHQANdfwL6qNGopBF99IscPTvCy95HSH1_kIF3eKjFDSLP0_iUUT0z8hiw"),
}
if _, store, err := c.Store(
pkg.ID{},
makeGarbage,
new(pkg.Checksum),
true,
); !reflect.DeepEqual(err, wantErrMismatch) {
t.Fatalf("Store: error = %v, want %v", err, wantErrMismatch)
} else if !store {
t.Fatal("Store did not store nonpresent entry")
}
}, pkg.MustDecode("N7dntFYbOq9V4iC-rjAQ-By6ofPIQVZkA8V0r0G07M_sdB7Zh42Ttrspsc38ioYa")},
}
checkWithCache(t, testCases)
}