security/hakurei
6
3
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 11 Wiki Activity

internal/pkg: parallelise scrub
All checks were successful
Test / Create distribution (push) Successful in 47s
Details
Test / ShareFS (push) Successful in 5m8s
Details
Test / Sandbox (race detector) (push) Successful in 5m28s
Details
Test / Hpkg (push) Successful in 5m39s
Details
Test / Hakurei (push) Successful in 6m3s
Details
Test / Hakurei (race detector) (push) Successful in 8m6s
Details
Test / Sandbox (push) Successful in 1m40s
Details
Test / Flake checks (push) Successful in 1m43s
Details

Browse Source 
This significantly improves scrubbing performance. Since the cache directory structure is friendly to simultaneous access, this is possible without synchronisation.

Signed-off-by: Ophestra <cat@gensokyo.uk>
This commit is contained in:
Ophestra
2026-01-16 02:09:12 +09:00
parent 3499a82785
commit 5936e6a4aa
4 changed files with 296 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
internal/pkg/pkg_test.go
View File

@@ -22,6 +22,7 @@ import (
"hakurei.app/container"
"hakurei.app/container/check"
"hakurei.app/container/fhs"
"hakurei.app/container/stub"
"hakurei.app/internal/pkg"
"hakurei.app/message"
@@ -281,7 +282,7 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) {
tc.early(t, base)
}
tc.f(t, base, c)
scrubFunc = c.Scrub
scrubFunc = func() error { return c.Scrub(1 << 7) }
}
var restoreTemp bool
@@ -293,7 +294,8 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) {
restoreTemp = true
}
if checksum, err := pkg.HashDir(base); err != nil {
var checksum pkg.Checksum
if err := pkg.HashDir(&checksum, base); err != nil {
t.Fatalf("HashDir: error = %v", err)
} else if checksum != tc.want {
t.Fatalf("HashDir: %v", &pkg.ChecksumMismatchError{
@@ -316,7 +318,7 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) {
}
// validate again to make sure scrub did not condemn anything
if checksum, err := pkg.HashDir(base); err != nil {
if err := pkg.HashDir(&checksum, base); err != nil {
t.Fatalf("HashDir: error = %v", err)
} else if checksum != tc.want {
t.Fatalf("(scrubbed) HashDir: %v", &pkg.ChecksumMismatchError{
@@ -364,6 +366,49 @@ func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) {
}
}
// newWantScrubError returns the address to a new [ScrubError] for base.
func newWantScrubError(base *check.Absolute) *pkg.ScrubError {
return &pkg.ScrubError{
ChecksumMismatches: []pkg.ChecksumMismatchError{
{Got: pkg.MustDecode(
"vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX",
), Want: pkg.Checksum{0xff, 0}},
},
DanglingIdentifiers: []pkg.ID{
{0xfe, 0},
{0xfe, 0xfe},
{0xfe, 0xff},
},
Errs: map[unique.Handle[string]][]error{
base.Append("checksum", "__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").Handle(): {
pkg.InvalidFileModeError(fs.ModeSymlink),
},
base.Append("checksum", "invalid").Handle(): {
base64.CorruptInputError(4),
},
base.Append("nonexistent").Handle(): {
base64.CorruptInputError(8),
},
base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).Handle(): {
&os.PathError{
Op: "readlink",
Path: base.Append(
"identifier",
pkg.Encode(pkg.ID{0xfe, 0xff}),
).String(),
Err: syscall.EINVAL,
},
},
base.Append("identifier", "invalid").Handle(): {
base64.CorruptInputError(4),
},
},
}
}
func TestCache(t *testing.T) {
t.Parallel()
@@ -744,9 +789,11 @@ func TestCache(t *testing.T) {
})
wantErrScrub := &pkg.ScrubError{
Errs: []error{errors.New("scrub began with pending artifacts")},
Errs: map[unique.Handle[string]][]error{
base.Handle(): {errors.New("scrub began with pending artifacts")},
},
}
if err := c.Scrub(); !reflect.DeepEqual(err, wantErrScrub) {
if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErrScrub) {
t.Fatalf("Scrub: error = %#v, want %#v", err, wantErrScrub)
}
@@ -800,30 +847,8 @@ func TestCache(t *testing.T) {
}
}
wantErr := &pkg.ScrubError{
ChecksumMismatches: []pkg.ChecksumMismatchError{
{Got: pkg.MustDecode(
"vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX",
), Want: pkg.Checksum{0xff, 0}},
},
DanglingIdentifiers: []pkg.ID{
{0xfe, 0},
{0xfe, 0xfe},
{0xfe, 0xff},
},
Errs: []error{
pkg.InvalidFileModeError(fs.ModeSymlink),
base64.CorruptInputError(4),
base64.CorruptInputError(8),
&os.PathError{
Op: "readlink",
Path: base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).String(),
Err: syscall.EINVAL,
},
base64.CorruptInputError(4),
},
}
if err := c.Scrub(); !reflect.DeepEqual(err, wantErr) {
wantErr := newWantScrubError(base)
if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErr) {
t.Fatalf("Scrub: error =\n%s\nwant\n%s", err, wantErr)
}
}, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")},
@@ -890,6 +915,48 @@ func TestScrubError(t *testing.T) {
want string
unwrap []error
}{
{"sample", *newWantScrubError(
fhs.AbsVarLib.Append("cure"),
), `checksum mismatches:
got vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX instead of _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
dangling identifiers:
_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
_v4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
errors during scrub:
/var/lib/cure/checksum/__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
artifact did not produce a regular file or directory
/var/lib/cure/checksum/invalid:
illegal base64 data at input byte 4
/var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
readlink /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: invalid argument
/var/lib/cure/identifier/invalid:
illegal base64 data at input byte 4
/var/lib/cure/nonexistent:
illegal base64 data at input byte 8
`, []error{
&pkg.ChecksumMismatchError{Got: pkg.MustDecode(
"vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX",
), Want: pkg.Checksum{0xff, 0}},
pkg.InvalidFileModeError(fs.ModeSymlink),
base64.CorruptInputError(4),
&os.PathError{
Op: "readlink",
Path: fhs.AbsVarLib.Append("cure").Append(
"identifier",
pkg.Encode(pkg.ID{0xfe, 0xff}),
).String(),
Err: syscall.EINVAL,
},
base64.CorruptInputError(4),
base64.CorruptInputError(8),
}},
{"full", pkg.ScrubError{
ChecksumMismatches: []pkg.ChecksumMismatchError{
{Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")},
@@ -898,10 +965,12 @@ func TestScrubError(t *testing.T) {
(pkg.ID)(bytes.Repeat([]byte{0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f}, 8)),
(pkg.ID)(bytes.Repeat([]byte{0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde}, 8)),
},
Errs: []error{
stub.UniqueError(0xcafe),
stub.UniqueError(0xbad),
stub.UniqueError(0xff),
Errs: map[unique.Handle[string]][]error{
unique.Make("/proc/nonexistent"): {
stub.UniqueError(0xcafe),
stub.UniqueError(0xbad),
stub.UniqueError(0xff),
},
},
}, `checksum mismatches:
got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA instead of CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN
@@ -911,9 +980,10 @@ deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef
cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe
errors during scrub:
unique error 51966 injected by the test suite
unique error 2989 injected by the test suite
unique error 255 injected by the test suite
/proc/nonexistent:
unique error 51966 injected by the test suite
unique error 2989 injected by the test suite
unique error 255 injected by the test suite
`, []error{
&pkg.ChecksumMismatchError{Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")},
stub.UniqueError(0xcafe),