diff --git a/internal/pkg/dir_test.go b/internal/pkg/dir_test.go index fe3daa3..63477ff 100644 --- a/internal/pkg/dir_test.go +++ b/internal/pkg/dir_test.go @@ -20,68 +20,136 @@ func TestFlatten(t *testing.T) { sum pkg.Checksum }{ {"empty", fstest.MapFS{ - ".": {Mode: 020000000700}, - "checksum": {Mode: 020000000700}, - "identifier": {Mode: 020000000700}, - "work": {Mode: 020000000700}, + ".": {Mode: fs.ModeDir | 0700}, + "checksum": {Mode: fs.ModeDir | 0700}, + "identifier": {Mode: fs.ModeDir | 0700}, + "work": {Mode: fs.ModeDir | 0700}, }, []pkg.FlatEntry{ - {Mode: 020000000700, Path: "."}, - {Mode: 020000000700, Path: "checksum"}, - {Mode: 020000000700, Path: "identifier"}, - {Mode: 020000000700, Path: "work"}, + {Mode: fs.ModeDir | 0700, Path: "."}, + {Mode: fs.ModeDir | 0700, Path: "checksum"}, + {Mode: fs.ModeDir | 0700, Path: "identifier"}, + {Mode: fs.ModeDir | 0700, Path: "work"}, }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")}, {"sample cache file", fstest.MapFS{ - ".": {Mode: 020000000700}, + ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: 020000000700}, + "checksum": {Mode: fs.ModeDir | 0700}, "checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}}, "checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, - "identifier": {Mode: 020000000700}, + "identifier": {Mode: fs.ModeDir | 0700}, "identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}}, "identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, "identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, "identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, - "work": {Mode: 020000000700}, + "work": {Mode: fs.ModeDir | 0700}, }, []pkg.FlatEntry{ - {Mode: 020000000700, Path: "."}, + {Mode: fs.ModeDir | 0700, Path: "."}, - {Mode: 020000000700, Path: "checksum"}, + {Mode: fs.ModeDir | 0700, Path: "checksum"}, {Mode: 0400, Path: "checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, {Mode: 0400, Path: "checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", Data: []byte{0}}, - {Mode: 020000000700, Path: "identifier"}, + {Mode: fs.ModeDir | 0700, Path: "identifier"}, {Mode: 0400, Path: "identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, {Mode: 0400, Path: "identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, {Mode: 0400, Path: "identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, {Mode: 0400, Path: "identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", Data: []byte{0}}, - {Mode: 020000000700, Path: "work"}, + {Mode: fs.ModeDir | 0700, Path: "work"}, }, pkg.MustDecode("G4u4W77C3u46oSAzwPTERKbS9h76iIvcd7Zl8p8Y6hTMb4_QGpH0Glg_DIJg-Usa")}, {"sample load or store", fstest.MapFS{ - ".": {Mode: 020000000700}, + ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: 020000000700}, + "checksum": {Mode: fs.ModeDir | 0700}, "checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")}, - "identifier": {Mode: 020000000700}, + "identifier": {Mode: fs.ModeDir | 0700}, "identifier/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")}, - "work": {Mode: 020000000700}, + "work": {Mode: fs.ModeDir | 0700}, }, []pkg.FlatEntry{ - {Mode: 020000000700, Path: "."}, + {Mode: fs.ModeDir | 0700, Path: "."}, - {Mode: 020000000700, Path: "checksum"}, + {Mode: fs.ModeDir | 0700, Path: "checksum"}, {Mode: 0400, Path: "checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU", Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")}, - {Mode: 020000000700, Path: "identifier"}, + {Mode: fs.ModeDir | 0700, Path: "identifier"}, {Mode: 0400, Path: "identifier/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU", Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")}, - {Mode: 020000000700, Path: "work"}, + {Mode: fs.ModeDir | 0700, Path: "work"}, }, pkg.MustDecode("bAJdeuI2cITlff5uIDbMKIiaW6zB-_XGinOMOMjNmltL3YAanXLiV64_jsXeQMXn")}, + + {"sample directory step simple", fstest.MapFS{ + ".": {Mode: fs.ModeDir | 0700}, + + "check": {Mode: 0400, Data: []byte{0, 0}}, + + "lib": {Mode: fs.ModeDir | 0700}, + "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, + + "lib/pkgconfig": {Mode: fs.ModeDir | 0700}, + }, []pkg.FlatEntry{ + {Mode: fs.ModeDir | 0700, Path: "."}, + + {Mode: 0400, Path: "check", Data: []byte{0x0, 0x0}}, + + {Mode: fs.ModeDir | 0700, Path: "lib"}, + {Mode: fs.ModeSymlink | 0777, Path: "lib/libedac.so", Data: []byte("/proc/nonexistent/libedac.so")}, + + {Mode: fs.ModeDir | 0700, Path: "lib/pkgconfig"}, + }, pkg.MustDecode("1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, + + {"sample directory step garbage", fstest.MapFS{ + ".": {Mode: fs.ModeDir | 0700}, + + "lib": {Mode: fs.ModeDir | 0500}, + "lib/check": {Mode: 0400, Data: []byte{}}, + + "lib/pkgconfig": {Mode: fs.ModeDir | 0500}, + }, []pkg.FlatEntry{ + {Mode: fs.ModeDir | 0700, Path: "."}, + + {Mode: fs.ModeDir | 0500, Path: "lib"}, + {Mode: 0400, Path: "lib/check", Data: []byte{}}, + + {Mode: fs.ModeDir | 0500, Path: "lib/pkgconfig"}, + }, pkg.MustDecode("GbjlYMcHQANdfwL6qNGopBF99IscPTvCy95HSH1_kIF3eKjFDSLP0_iUUT0z8hiw")}, + + {"sample directory", fstest.MapFS{ + ".": {Mode: fs.ModeDir | 0700}, + + "checksum": {Mode: fs.ModeDir | 0700}, + "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0700}, + "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, + "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0700}, + "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0700}, + "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: 01000000777, Data: []byte("/proc/nonexistent/libedac.so")}, + + "identifier": {Mode: fs.ModeDir | 0700}, + "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: 01000000777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, + "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: 01000000777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, + + "work": {Mode: fs.ModeDir | 0700}, + }, []pkg.FlatEntry{ + {Mode: fs.ModeDir | 0700, Path: "."}, + + {Mode: fs.ModeDir | 0700, Path: "checksum"}, + {Mode: fs.ModeDir | 0700, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP"}, + {Mode: 0400, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check", Data: []byte{0, 0}}, + {Mode: fs.ModeDir | 0700, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib"}, + {Mode: fs.ModeSymlink | 0777, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so", Data: []byte("/proc/nonexistent/libedac.so")}, + {Mode: fs.ModeDir | 0700, Path: "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig"}, + + {Mode: fs.ModeDir | 0700, Path: "identifier"}, + {Mode: fs.ModeSymlink | 0777, Path: "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, + {Mode: fs.ModeSymlink | 0777, Path: "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa", Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, + + {Mode: fs.ModeDir | 0700, Path: "work"}, + }, pkg.MustDecode("N7dntFYbOq9V4iC-rjAQ-By6ofPIQVZkA8V0r0G07M_sdB7Zh42Ttrspsc38ioYa")}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go index 64df5df..e8547d1 100644 --- a/internal/pkg/pkg.go +++ b/internal/pkg/pkg.go @@ -8,7 +8,10 @@ import ( "encoding/binary" "errors" "io" + "io/fs" "os" + "path" + "path/filepath" "slices" "sync" @@ -190,6 +193,100 @@ func (c *Cache) pathnameIdent(id *ID) *check.Absolute { ) } +// Store looks up an identifier, and if it is not present, calls makeArtifact +// with a private working directory and stores its result instead. An optional +// checksum can be passed via the result buffer which is used to validate the +// produced directory. +func (c *Cache) Store( + id ID, + makeArtifact func(work *check.Absolute) error, + buf *Checksum, + validate bool, +) ( + pathname *check.Absolute, + store bool, + err error, +) { + pathname = c.pathnameIdent(&id) + c.mu.Lock() + defer c.mu.Unlock() + + _, err = os.Lstat(pathname.String()) + if err == nil || !errors.Is(err, os.ErrNotExist) { + return + } + store = true + + var ( + workPathname *check.Absolute + workPathnameRaw string + ) + if workPathnameRaw, err = os.MkdirTemp( + c.base.Append(dirWork).String(), + path.Base(pathname.String()+".*"), + ); err != nil { + return + } else if workPathname, err = check.NewAbs(workPathnameRaw); err != nil { + return + } + defer func() { + if err != nil { + chmodErr := filepath.WalkDir(workPathname.String(), func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return os.Chmod(path, 0700) + } + return nil + }) + removeErr := os.RemoveAll(workPathname.String()) + if chmodErr != nil || removeErr != nil { + err = errors.Join(err, chmodErr, removeErr) + } else if errors.Is(err, os.ErrExist) { + // two artifacts may be backed by the same file + err = nil + } + } + }() + if err = os.Chmod(workPathname.String(), 0700); err != nil { + return + } + + if err = makeArtifact(workPathname); err != nil { + return + } + var checksum Checksum + if checksum, err = HashDir(workPathname); err != nil { + return + } + if validate { + if checksum != *buf { + err = &ChecksumMismatchError{checksum, *buf} + return + } + } else { + *buf = checksum + } + + checksumPathname := c.pathname(&checksum) + if err = os.Rename( + workPathname.String(), + checksumPathname.String(), + ); err != nil { + if !errors.Is(err, os.ErrExist) { + return + } + } + if linkErr := os.Symlink( + "../"+dirChecksum+"/"+path.Base(checksumPathname.String()), + pathname.String(), + ); linkErr != nil { + err = linkErr + } + return +} + // storeFile stores the contents of a [File]. An optional checksum can be // passed via the result buffer which is used to validate the submitted data. // diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go index f295ef0..46084ff 100644 --- a/internal/pkg/pkg_test.go +++ b/internal/pkg/pkg_test.go @@ -3,6 +3,7 @@ package pkg_test import ( "bytes" "crypto/sha512" + "encoding/binary" "io/fs" "net/http" "os" @@ -303,6 +304,158 @@ func TestCache(t *testing.T) { t.Fatalf("LoadFile: error = %#v, want %#v", err, wantErrNonexistentZero) } }, pkg.MustDecode("G4u4W77C3u46oSAzwPTERKbS9h76iIvcd7Zl8p8Y6hTMb4_QGpH0Glg_DIJg-Usa")}, + + {"directory", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { + id := pkg.KindTar.Ident( + binary.LittleEndian.AppendUint64(nil, 1), + stubArtifact{pkg.KindHTTP, testdataChecksum}, + ) + makeSample := func(work *check.Absolute) error { + if err := os.WriteFile( + work.Append("check").String(), + []byte{0, 0}, + 0400, + ); err != nil { + return err + } + + if err := os.MkdirAll(work.Append( + "lib", + "pkgconfig", + ).String(), 0700); err != nil { + return err + } + + return os.Symlink( + "/proc/nonexistent/libedac.so", + work.Append( + "lib", + "libedac.so", + ).String(), + ) + } + wantChecksum := pkg.MustDecode( + "1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP", + ) + wantPathname := base.Append( + "identifier", + pkg.Encode(id), + ) + + if pathname, store, err := c.Store( + id, + makeSample, + &wantChecksum, + true, + ); err != nil { + t.Fatalf("Store: error = %v", err) + } else if !store { + t.Fatal("Store did not store nonpresent entry") + } else if !pathname.Is(wantPathname) { + t.Fatalf("Store: pathname = %q, want %q", pathname, wantPathname) + } + + // check lookup + if pathname, store, err := c.Store( + id, + nil, + &wantChecksum, + true, + ); err != nil { + t.Fatalf("Store: error = %v", err) + } else if store { + t.Fatal("Store stored over present entry") + } else if !pathname.Is(wantPathname) { + t.Fatalf("Store: pathname = %q, want %q", pathname, wantPathname) + } + + // check exist + id0 := pkg.KindTar.Ident( + binary.LittleEndian.AppendUint64(nil, 1), + stubArtifact{pkg.KindHTTP, pkg.ID{}}, + ) + wantPathname0 := base.Append( + "identifier", + pkg.Encode(id0), + ) + if pathname, store, err := c.Store( + id0, + makeSample, + &wantChecksum, + true, + ); err != nil { + t.Fatalf("Store: error = %v", err) + } else if !store { + t.Fatal("Store did not store nonpresent entry") + } else if !pathname.Is(wantPathname0) { + t.Fatalf("Store: pathname = %q, want %q", pathname, wantPathname0) + } + + var wantErrMakeGarbage error + makeGarbage := func(work *check.Absolute) error { + mode := fs.FileMode(0) + if wantErrMakeGarbage == nil { + mode = 0500 + } + + if err := os.MkdirAll(work.Append( + "lib", + "pkgconfig", + ).String(), 0700); err != nil { + return err + } + + if err := os.WriteFile(work.Append( + "lib", + "check", + ).String(), nil, 0400&mode); err != nil { + return err + } + + if err := os.Chmod(work.Append( + "lib", + "pkgconfig", + ).String(), 0500&mode); err != nil { + return err + } + if err := os.Chmod(work.Append( + "lib", + ).String(), 0500&mode); err != nil { + return err + } + + return wantErrMakeGarbage + } + + // check makeArtifact fault + wantErrMakeGarbage = stub.UniqueError(0xcafe) + if _, store, err := c.Store( + pkg.ID{}, + makeGarbage, + nil, + false, + ); !reflect.DeepEqual(err, wantErrMakeGarbage) { + t.Fatalf("Store: error = %#v, want %#v", err, wantErrMakeGarbage) + } else if !store { + t.Fatal("Store did not store nonpresent entry") + } + + // checksum mismatch + wantErrMakeGarbage = nil + wantErrMismatch := &pkg.ChecksumMismatchError{ + Got: pkg.MustDecode("GbjlYMcHQANdfwL6qNGopBF99IscPTvCy95HSH1_kIF3eKjFDSLP0_iUUT0z8hiw"), + } + if _, store, err := c.Store( + pkg.ID{}, + makeGarbage, + new(pkg.Checksum), + true, + ); !reflect.DeepEqual(err, wantErrMismatch) { + t.Fatalf("Store: error = %v, want %v", err, wantErrMismatch) + } else if !store { + t.Fatal("Store did not store nonpresent entry") + } + }, pkg.MustDecode("N7dntFYbOq9V4iC-rjAQ-By6ofPIQVZkA8V0r0G07M_sdB7Zh42Ttrspsc38ioYa")}, } checkWithCache(t, testCases) }