diff --git a/cmd/hakurei/print.go b/cmd/hakurei/print.go index 3b4730f..aa7a26a 100644 --- a/cmd/hakurei/print.go +++ b/cmd/hakurei/print.go @@ -174,9 +174,6 @@ func printPs(output io.Writer, now time.Time, s state.Store, short, flagJSON boo } else { entries = e } - if err := s.Close(); err != nil { - log.Printf("cannot close store: %v", err) - } if !short && flagJSON { es := make(map[string]*hst.State, len(entries)) diff --git a/internal/app/process.go b/internal/app/process.go index 8e18ae6..a0950c2 100644 --- a/internal/app/process.go +++ b/internal/app/process.go @@ -192,12 +192,6 @@ func (ms mainState) beforeExit(isFault bool) { } else if ms.uintptr&mainNeedsDestroy != 0 { panic("unreachable") } - - if ms.store != nil { - if err := ms.store.Close(); err != nil { - perror(err, "close state store") - } - } } // fatal calls printMessageError, performs necessary cleanup, followed by a call to [os.Exit](1). diff --git a/internal/app/state/multi.go b/internal/app/state/multi.go index c9096ac..9824a8e 100644 --- a/internal/app/state/multi.go +++ b/internal/app/state/multi.go @@ -9,12 +9,15 @@ import ( "path" "strconv" "sync" - "syscall" "hakurei.app/hst" + "hakurei.app/internal/lockedfile" "hakurei.app/message" ) +// multiLockFileName is the name of the file backing [lockedfile.Mutex] of a multiBackend. +const multiLockFileName = "lock" + // fine-grained locking and access type multiStore struct { base string @@ -44,19 +47,18 @@ func (s *multiStore) Do(identity int, f func(c Cursor)) (bool, error) { return false, &hst.AppError{Step: "create store segment directory", Err: err} } - // open locker file - if l, err := os.OpenFile(b.path+".lock", os.O_RDWR|os.O_CREATE, 0600); err != nil { - s.backends.CompareAndDelete(identity, b) - return false, &hst.AppError{Step: "open store segment lock file", Err: err} - } else { - b.lockfile = l - } + // set up file-based mutex + b.lockfile = lockedfile.MutexAt(path.Join(b.path, multiLockFileName)) + b.mu.Unlock() } // lock backend - if err := b.lockFile(); err != nil { + if unlock, err := b.lockfile.Lock(); err != nil { return false, &hst.AppError{Step: "lock store segment", Err: err} + } else { + // unlock backend after Do is complete + defer unlock() } // expose backend methods without exporting the pointer @@ -66,10 +68,6 @@ func (s *multiStore) Do(identity int, f func(c Cursor)) (bool, error) { // disable access to the backend on a best-effort basis c.multiBackend = nil - // unlock backend - if err := b.unlockFile(); err != nil { - return true, &hst.AppError{Step: "unlock store segment", Err: err} - } return true, nil } @@ -108,59 +106,17 @@ func (s *multiStore) List() ([]int, error) { return append([]int(nil), aidsBuf...), nil } -func (s *multiStore) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - - var errs []error - s.backends.Range(func(_, value any) bool { - b := value.(*multiBackend) - errs = append(errs, b.close()) - return true - }) - - return errors.Join(errs...) -} - type multiBackend struct { path string // created/opened by prepare - lockfile *os.File + lockfile *lockedfile.Mutex mu sync.RWMutex } func (b *multiBackend) filename(id *hst.ID) string { return path.Join(b.path, id.String()) } -func (b *multiBackend) lockFileAct(lt int) (err error) { - op := "LockAct" - switch lt { - case syscall.LOCK_EX: - op = "Lock" - case syscall.LOCK_UN: - op = "Unlock" - } - - for { - err = syscall.Flock(int(b.lockfile.Fd()), lt) - if !errors.Is(err, syscall.EINTR) { - break - } - } - if err != nil { - return &fs.PathError{ - Op: op, - Path: b.lockfile.Name(), - Err: err, - } - } - return nil -} - -func (b *multiBackend) lockFile() error { return b.lockFileAct(syscall.LOCK_EX) } -func (b *multiBackend) unlockFile() error { return b.lockFileAct(syscall.LOCK_UN) } - // reads all launchers in simpleBackend // file contents are ignored if decode is false func (b *multiBackend) load(decode bool) (map[hst.ID]*hst.State, error) { @@ -184,6 +140,11 @@ func (b *multiBackend) load(decode bool) (map[hst.ID]*hst.State, error) { return nil, fmt.Errorf("unexpected directory %q in store", e.Name()) } + // skip lock file + if e.Name() == multiLockFileName { + continue + } + var id hst.ID if err := id.UnmarshalText([]byte(e.Name())); err != nil { return nil, &hst.AppError{Step: "parse state key", Err: err} @@ -268,17 +229,6 @@ func (b *multiBackend) Len() (int, error) { return len(rn), nil } -func (b *multiBackend) close() error { - b.mu.Lock() - defer b.mu.Unlock() - - err := b.lockfile.Close() - if err == nil || errors.Is(err, os.ErrInvalid) || errors.Is(err, os.ErrClosed) { - return nil - } - return &hst.AppError{Step: "close lock file", Err: err} -} - // NewMulti returns an instance of the multi-file store. func NewMulti(msg message.Msg, runDir string) Store { return &multiStore{ diff --git a/internal/app/state/state.go b/internal/app/state/state.go index ef783bf..c5238d1 100644 --- a/internal/app/state/state.go +++ b/internal/app/state/state.go @@ -19,9 +19,6 @@ type Store interface { // List queries the store and returns a list of identities known to the store. // Note that some or all returned identities might not have any active apps. List() (identities []int, err error) - - // Close releases any resources held by Store. - Close() error } // Cursor provides access to the store of an identity. diff --git a/internal/app/state/state_test.go b/internal/app/state/state_test.go index 2e191f3..a334c34 100644 --- a/internal/app/state/state_test.go +++ b/internal/app/state/state_test.go @@ -62,62 +62,49 @@ func testStore(t *testing.T, s state.Store) { }) } - t.Run("insert entry checked", func(t *testing.T) { - insert(insertEntryChecked, 0) - check(insertEntryChecked, 0) - }) + // insert entry checked + insert(insertEntryChecked, 0) + check(insertEntryChecked, 0) - t.Run("insert entry unchecked", func(t *testing.T) { - insert(insertEntryNoCheck, 0) - }) + // insert entry unchecked + insert(insertEntryNoCheck, 0) - t.Run("insert entry different identity", func(t *testing.T) { - insert(insertEntryOtherApp, 1) - check(insertEntryOtherApp, 1) - }) + // insert entry different identity + insert(insertEntryOtherApp, 1) + check(insertEntryOtherApp, 1) - t.Run("check previous insertion", func(t *testing.T) { - check(insertEntryNoCheck, 0) - }) + // check previous insertion + check(insertEntryNoCheck, 0) - t.Run("list identities", func(t *testing.T) { - if identities, err := s.List(); err != nil { - t.Fatalf("List: error = %v", err) - } else { - slices.Sort(identities) - want := []int{0, 1} - if !slices.Equal(identities, want) { - t.Fatalf("List() = %#v, want %#v", identities, want) - } + // list identities + if identities, err := s.List(); err != nil { + t.Fatalf("List: error = %v", err) + } else { + slices.Sort(identities) + want := []int{0, 1} + if !slices.Equal(identities, want) { + t.Fatalf("List() = %#v, want %#v", identities, want) + } + } + + // join store + if entries, err := state.Join(s); err != nil { + t.Fatalf("Join: error = %v", err) + } else if len(entries) != 3 { + t.Fatalf("Join(s) = %#v", entries) + } + + // clear identity 1 + do(1, func(c state.Cursor) { + if err := c.Destroy(tc[insertEntryOtherApp].ID); err != nil { + t.Fatalf("Destroy: error = %v", err) } }) - - t.Run("join store", func(t *testing.T) { - if entries, err := state.Join(s); err != nil { - t.Fatalf("Join: error = %v", err) - } else if len(entries) != 3 { - t.Fatalf("Join(s) = %#v", entries) - } - }) - - t.Run("clear identity 1", func(t *testing.T) { - do(1, func(c state.Cursor) { - if err := c.Destroy(tc[insertEntryOtherApp].ID); err != nil { - t.Fatalf("Destroy: error = %v", err) - } - }) - do(1, func(c state.Cursor) { - if l, err := c.Len(); err != nil { - t.Fatalf("Len: error = %v", err) - } else if l != 0 { - t.Fatalf("Len: %d, want 0", l) - } - }) - }) - - t.Run("close store", func(t *testing.T) { - if err := s.Close(); err != nil { - t.Fatalf("Close: error = %v", err) + do(1, func(c state.Cursor) { + if l, err := c.Len(); err != nil { + t.Fatalf("Len: error = %v", err) + } else if l != 0 { + t.Fatalf("Len: %d, want 0", l) } }) } diff --git a/internal/lockedfile/internal/filelock/filelock.go b/internal/lockedfile/internal/filelock/filelock.go new file mode 100644 index 0000000..d373318 --- /dev/null +++ b/internal/lockedfile/internal/filelock/filelock.go @@ -0,0 +1,83 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package filelock provides a platform-independent API for advisory file +// locking. Calls to functions in this package on platforms that do not support +// advisory locks will return errors for which IsNotSupported returns true. +package filelock + +import ( + "errors" + "io/fs" +) + +// A File provides the minimal set of methods required to lock an open file. +// File implementations must be usable as map keys. +// The usual implementation is *os.File. +type File interface { + // Name returns the name of the file. + Name() string + + // Fd returns a valid file descriptor. + // (If the File is an *os.File, it must not be closed.) + Fd() uintptr + + // Stat returns the FileInfo structure describing file. + Stat() (fs.FileInfo, error) +} + +// Lock places an advisory write lock on the file, blocking until it can be +// locked. +// +// If Lock returns nil, no other process will be able to place a read or write +// lock on the file until this process exits, closes f, or calls Unlock on it. +// +// If f's descriptor is already read- or write-locked, the behavior of Lock is +// unspecified. +// +// Closing the file may or may not release the lock promptly. Callers should +// ensure that Unlock is always called when Lock succeeds. +func Lock(f File) error { + return lock(f, writeLock) +} + +// RLock places an advisory read lock on the file, blocking until it can be locked. +// +// If RLock returns nil, no other process will be able to place a write lock on +// the file until this process exits, closes f, or calls Unlock on it. +// +// If f is already read- or write-locked, the behavior of RLock is unspecified. +// +// Closing the file may or may not release the lock promptly. Callers should +// ensure that Unlock is always called if RLock succeeds. +func RLock(f File) error { + return lock(f, readLock) +} + +// Unlock removes an advisory lock placed on f by this process. +// +// The caller must not attempt to unlock a file that is not locked. +func Unlock(f File) error { + return unlock(f) +} + +// String returns the name of the function corresponding to lt +// (Lock, RLock, or Unlock). +func (lt lockType) String() string { + switch lt { + case readLock: + return "RLock" + case writeLock: + return "Lock" + default: + return "Unlock" + } +} + +// IsNotSupported returns a boolean indicating whether the error is known to +// report that a function is not supported (possibly for a specific input). +// It is satisfied by errors.ErrUnsupported as well as some syscall errors. +func IsNotSupported(err error) bool { + return errors.Is(err, errors.ErrUnsupported) +} diff --git a/internal/lockedfile/internal/filelock/filelock_fcntl.go b/internal/lockedfile/internal/filelock/filelock_fcntl.go new file mode 100644 index 0000000..8a62839 --- /dev/null +++ b/internal/lockedfile/internal/filelock/filelock_fcntl.go @@ -0,0 +1,210 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || (solaris && !illumos) + +// This code implements the filelock API using POSIX 'fcntl' locks, which attach +// to an (inode, process) pair rather than a file descriptor. To avoid unlocking +// files prematurely when the same file is opened through different descriptors, +// we allow only one read-lock at a time. +// +// Most platforms provide some alternative API, such as an 'flock' system call +// or an F_OFD_SETLK command for 'fcntl', that allows for better concurrency and +// does not require per-inode bookkeeping in the application. + +package filelock + +import ( + "errors" + "io" + "io/fs" + "math/rand" + "sync" + "syscall" + "time" +) + +type lockType int16 + +const ( + readLock lockType = syscall.F_RDLCK + writeLock lockType = syscall.F_WRLCK +) + +type inode = uint64 // type of syscall.Stat_t.Ino + +type inodeLock struct { + owner File + queue []<-chan File +} + +var ( + mu sync.Mutex + inodes = map[File]inode{} + locks = map[inode]inodeLock{} +) + +func lock(f File, lt lockType) (err error) { + // POSIX locks apply per inode and process, and the lock for an inode is + // released when *any* descriptor for that inode is closed. So we need to + // synchronize access to each inode internally, and must serialize lock and + // unlock calls that refer to the same inode through different descriptors. + fi, err := f.Stat() + if err != nil { + return err + } + ino := fi.Sys().(*syscall.Stat_t).Ino + + mu.Lock() + if i, dup := inodes[f]; dup && i != ino { + mu.Unlock() + return &fs.PathError{ + Op: lt.String(), + Path: f.Name(), + Err: errors.New("inode for file changed since last Lock or RLock"), + } + } + inodes[f] = ino + + var wait chan File + l := locks[ino] + if l.owner == f { + // This file already owns the lock, but the call may change its lock type. + } else if l.owner == nil { + // No owner: it's ours now. + l.owner = f + } else { + // Already owned: add a channel to wait on. + wait = make(chan File) + l.queue = append(l.queue, wait) + } + locks[ino] = l + mu.Unlock() + + if wait != nil { + wait <- f + } + + // Spurious EDEADLK errors arise on platforms that compute deadlock graphs at + // the process, rather than thread, level. Consider processes P and Q, with + // threads P.1, P.2, and Q.3. The following trace is NOT a deadlock, but will be + // reported as a deadlock on systems that consider only process granularity: + // + // P.1 locks file A. + // Q.3 locks file B. + // Q.3 blocks on file A. + // P.2 blocks on file B. (This is erroneously reported as a deadlock.) + // P.1 unlocks file A. + // Q.3 unblocks and locks file A. + // Q.3 unlocks files A and B. + // P.2 unblocks and locks file B. + // P.2 unlocks file B. + // + // These spurious errors were observed in practice on AIX and Solaris in + // cmd/go: see https://golang.org/issue/32817. + // + // We work around this bug by treating EDEADLK as always spurious. If there + // really is a lock-ordering bug between the interacting processes, it will + // become a livelock instead, but that's not appreciably worse than if we had + // a proper flock implementation (which generally does not even attempt to + // diagnose deadlocks). + // + // In the above example, that changes the trace to: + // + // P.1 locks file A. + // Q.3 locks file B. + // Q.3 blocks on file A. + // P.2 spuriously fails to lock file B and goes to sleep. + // P.1 unlocks file A. + // Q.3 unblocks and locks file A. + // Q.3 unlocks files A and B. + // P.2 wakes up and locks file B. + // P.2 unlocks file B. + // + // We know that the retry loop will not introduce a *spurious* livelock + // because, according to the POSIX specification, EDEADLK is only to be + // returned when “the lock is blocked by a lock from another process”. + // If that process is blocked on some lock that we are holding, then the + // resulting livelock is due to a real deadlock (and would manifest as such + // when using, for example, the flock implementation of this package). + // If the other process is *not* blocked on some other lock that we are + // holding, then it will eventually release the requested lock. + + nextSleep := 1 * time.Millisecond + const maxSleep = 500 * time.Millisecond + for { + err = setlkw(f.Fd(), lt) + if err != syscall.EDEADLK { + break + } + time.Sleep(nextSleep) + + nextSleep += nextSleep + if nextSleep > maxSleep { + nextSleep = maxSleep + } + // Apply 10% jitter to avoid synchronizing collisions when we finally unblock. + nextSleep += time.Duration((0.1*rand.Float64() - 0.05) * float64(nextSleep)) + } + + if err != nil { + unlock(f) + return &fs.PathError{ + Op: lt.String(), + Path: f.Name(), + Err: err, + } + } + + return nil +} + +func unlock(f File) error { + var owner File + + mu.Lock() + ino, ok := inodes[f] + if ok { + owner = locks[ino].owner + } + mu.Unlock() + + if owner != f { + panic("unlock called on a file that is not locked") + } + + err := setlkw(f.Fd(), syscall.F_UNLCK) + + mu.Lock() + l := locks[ino] + if len(l.queue) == 0 { + // No waiters: remove the map entry. + delete(locks, ino) + } else { + // The first waiter is sending us their file now. + // Receive it and update the queue. + l.owner = <-l.queue[0] + l.queue = l.queue[1:] + locks[ino] = l + } + delete(inodes, f) + mu.Unlock() + + return err +} + +// setlkw calls FcntlFlock with F_SETLKW for the entire file indicated by fd. +func setlkw(fd uintptr, lt lockType) error { + for { + err := syscall.FcntlFlock(fd, syscall.F_SETLKW, &syscall.Flock_t{ + Type: int16(lt), + Whence: io.SeekStart, + Start: 0, + Len: 0, // All bytes. + }) + if err != syscall.EINTR { + return err + } + } +} diff --git a/internal/lockedfile/internal/filelock/filelock_other.go b/internal/lockedfile/internal/filelock/filelock_other.go new file mode 100644 index 0000000..b16709e --- /dev/null +++ b/internal/lockedfile/internal/filelock/filelock_other.go @@ -0,0 +1,35 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !unix && !windows + +package filelock + +import ( + "errors" + "io/fs" +) + +type lockType int8 + +const ( + readLock = iota + 1 + writeLock +) + +func lock(f File, lt lockType) error { + return &fs.PathError{ + Op: lt.String(), + Path: f.Name(), + Err: errors.ErrUnsupported, + } +} + +func unlock(f File) error { + return &fs.PathError{ + Op: "Unlock", + Path: f.Name(), + Err: errors.ErrUnsupported, + } +} diff --git a/internal/lockedfile/internal/filelock/filelock_test.go b/internal/lockedfile/internal/filelock/filelock_test.go new file mode 100644 index 0000000..4e40739 --- /dev/null +++ b/internal/lockedfile/internal/filelock/filelock_test.go @@ -0,0 +1,209 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !js && !plan9 && !wasip1 + +package filelock_test + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "hakurei.app/container" + "hakurei.app/internal/lockedfile/internal/filelock" + "hakurei.app/internal/lockedfile/internal/testexec" +) + +func lock(t *testing.T, f *os.File) { + t.Helper() + err := filelock.Lock(f) + t.Logf("Lock(fd %d) = %v", f.Fd(), err) + if err != nil { + t.Fail() + } +} + +func rLock(t *testing.T, f *os.File) { + t.Helper() + err := filelock.RLock(f) + t.Logf("RLock(fd %d) = %v", f.Fd(), err) + if err != nil { + t.Fail() + } +} + +func unlock(t *testing.T, f *os.File) { + t.Helper() + err := filelock.Unlock(f) + t.Logf("Unlock(fd %d) = %v", f.Fd(), err) + if err != nil { + t.Fail() + } +} + +func mustTempFile(t *testing.T) (f *os.File, remove func()) { + t.Helper() + + base := filepath.Base(t.Name()) + f, err := os.CreateTemp("", base) + if err != nil { + t.Fatalf(`os.CreateTemp("", %q) = %v`, base, err) + } + t.Logf("fd %d = %s", f.Fd(), f.Name()) + + return f, func() { + f.Close() + os.Remove(f.Name()) + } +} + +func mustOpen(t *testing.T, name string) *os.File { + t.Helper() + + f, err := os.OpenFile(name, os.O_RDWR, 0) + if err != nil { + t.Fatalf("os.OpenFile(%q) = %v", name, err) + } + + t.Logf("fd %d = os.OpenFile(%q)", f.Fd(), name) + return f +} + +const ( + quiescent = 10 * time.Millisecond + probablyStillBlocked = 10 * time.Second +) + +func mustBlock(t *testing.T, op string, f *os.File) (wait func(*testing.T)) { + t.Helper() + + desc := fmt.Sprintf("%s(fd %d)", op, f.Fd()) + + done := make(chan struct{}) + go func() { + t.Helper() + switch op { + case "Lock": + lock(t, f) + case "RLock": + rLock(t, f) + default: + panic("invalid op: " + op) + } + close(done) + }() + + select { + case <-done: + t.Fatalf("%s unexpectedly did not block", desc) + return nil + + case <-time.After(quiescent): + t.Logf("%s is blocked (as expected)", desc) + return func(t *testing.T) { + t.Helper() + select { + case <-time.After(probablyStillBlocked): + t.Fatalf("%s is unexpectedly still blocked", desc) + case <-done: + } + } + } +} + +func TestLockExcludesLock(t *testing.T) { + t.Parallel() + + f, remove := mustTempFile(t) + defer remove() + + other := mustOpen(t, f.Name()) + defer other.Close() + + lock(t, f) + lockOther := mustBlock(t, "Lock", other) + unlock(t, f) + lockOther(t) + unlock(t, other) +} + +func TestLockExcludesRLock(t *testing.T) { + t.Parallel() + + f, remove := mustTempFile(t) + defer remove() + + other := mustOpen(t, f.Name()) + defer other.Close() + + lock(t, f) + rLockOther := mustBlock(t, "RLock", other) + unlock(t, f) + rLockOther(t) + unlock(t, other) +} + +func TestRLockExcludesOnlyLock(t *testing.T) { + t.Parallel() + + f, remove := mustTempFile(t) + defer remove() + rLock(t, f) + + f2 := mustOpen(t, f.Name()) + defer f2.Close() + + doUnlockTF := false + switch runtime.GOOS { + case "aix", "solaris": + // When using POSIX locks (as on Solaris), we can't safely read-lock the + // same inode through two different descriptors at the same time: when the + // first descriptor is closed, the second descriptor would still be open but + // silently unlocked. So a second RLock must block instead of proceeding. + lockF2 := mustBlock(t, "RLock", f2) + unlock(t, f) + lockF2(t) + default: + rLock(t, f2) + doUnlockTF = true + } + + other := mustOpen(t, f.Name()) + defer other.Close() + lockOther := mustBlock(t, "Lock", other) + + unlock(t, f2) + if doUnlockTF { + unlock(t, f) + } + lockOther(t) + unlock(t, other) +} + +func TestLockNotDroppedByExecCommand(t *testing.T) { + f, remove := mustTempFile(t) + defer remove() + + lock(t, f) + + other := mustOpen(t, f.Name()) + defer other.Close() + + // Some kinds of file locks are dropped when a duplicated or forked file + // descriptor is unlocked. Double-check that the approach used by os/exec does + // not accidentally drop locks. + cmd := testexec.CommandContext(t, t.Context(), container.MustExecutable(nil), "-test.run=^$") + if err := cmd.Run(); err != nil { + t.Fatalf("exec failed: %v", err) + } + + lockOther := mustBlock(t, "Lock", other) + unlock(t, f) + lockOther(t) + unlock(t, other) +} diff --git a/internal/lockedfile/internal/filelock/filelock_unix.go b/internal/lockedfile/internal/filelock/filelock_unix.go new file mode 100644 index 0000000..6f73b1b --- /dev/null +++ b/internal/lockedfile/internal/filelock/filelock_unix.go @@ -0,0 +1,40 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || illumos || linux || netbsd || openbsd + +package filelock + +import ( + "io/fs" + "syscall" +) + +type lockType int16 + +const ( + readLock lockType = syscall.LOCK_SH + writeLock lockType = syscall.LOCK_EX +) + +func lock(f File, lt lockType) (err error) { + for { + err = syscall.Flock(int(f.Fd()), int(lt)) + if err != syscall.EINTR { + break + } + } + if err != nil { + return &fs.PathError{ + Op: lt.String(), + Path: f.Name(), + Err: err, + } + } + return nil +} + +func unlock(f File) error { + return lock(f, syscall.LOCK_UN) +} diff --git a/internal/lockedfile/internal/filelock/filelock_windows.go b/internal/lockedfile/internal/filelock/filelock_windows.go new file mode 100644 index 0000000..647ee99 --- /dev/null +++ b/internal/lockedfile/internal/filelock/filelock_windows.go @@ -0,0 +1,57 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package filelock + +import ( + "internal/syscall/windows" + "io/fs" + "syscall" +) + +type lockType uint32 + +const ( + readLock lockType = 0 + writeLock lockType = windows.LOCKFILE_EXCLUSIVE_LOCK +) + +const ( + reserved = 0 + allBytes = ^uint32(0) +) + +func lock(f File, lt lockType) error { + // Per https://golang.org/issue/19098, “Programs currently expect the Fd + // method to return a handle that uses ordinary synchronous I/O.” + // However, LockFileEx still requires an OVERLAPPED structure, + // which contains the file offset of the beginning of the lock range. + // We want to lock the entire file, so we leave the offset as zero. + ol := new(syscall.Overlapped) + + err := windows.LockFileEx(syscall.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) + if err != nil { + return &fs.PathError{ + Op: lt.String(), + Path: f.Name(), + Err: err, + } + } + return nil +} + +func unlock(f File) error { + ol := new(syscall.Overlapped) + err := windows.UnlockFileEx(syscall.Handle(f.Fd()), reserved, allBytes, allBytes, ol) + if err != nil { + return &fs.PathError{ + Op: "Unlock", + Path: f.Name(), + Err: err, + } + } + return nil +} diff --git a/internal/lockedfile/internal/testexec/exec.go b/internal/lockedfile/internal/testexec/exec.go new file mode 100644 index 0000000..6adf4bf --- /dev/null +++ b/internal/lockedfile/internal/testexec/exec.go @@ -0,0 +1,43 @@ +package testexec + +import ( + "context" + "os/exec" + "syscall" + "testing" +) + +// CommandContext is like exec.CommandContext, but: +// - sends SIGQUIT instead of SIGKILL in its Cancel function +// - fails the test if the command does not complete before the context is canceled, and +// - sets a Cleanup function that verifies that the test did not leak a subprocess. +func CommandContext(t testing.TB, ctx context.Context, name string, args ...string) *exec.Cmd { + t.Helper() + + cmd := exec.CommandContext(ctx, name, args...) + cmd.Cancel = func() error { + if ctx.Err() == context.DeadlineExceeded { + // The command timed out due to running too close to the test's deadline. + // There is no way the test did that intentionally — it's too close to the + // wire! — so mark it as a test failure. That way, if the test expects the + // command to fail for some other reason, it doesn't have to distinguish + // between that reason and a timeout. + t.Errorf("test timed out while running command: %v", cmd) + } else { + // The command is being terminated due to ctx being canceled, but + // apparently not due to an explicit test deadline that we added. + // Log that information in case it is useful for diagnosing a failure, + // but don't actually fail the test because of it. + t.Logf("%v: terminating command: %v", ctx.Err(), cmd) + } + return cmd.Process.Signal(syscall.SIGQUIT) + } + + t.Cleanup(func() { + if cmd.Process != nil && cmd.ProcessState == nil { + t.Errorf("command was started, but test did not wait for it to complete: %v", cmd) + } + }) + + return cmd +} diff --git a/internal/lockedfile/lockedfile.go b/internal/lockedfile/lockedfile.go new file mode 100644 index 0000000..8bd2ffb --- /dev/null +++ b/internal/lockedfile/lockedfile.go @@ -0,0 +1,189 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lockedfile creates and manipulates files whose contents should only +// change atomically. +package lockedfile + +import ( + "fmt" + "io" + "io/fs" + "os" + "runtime" +) + +// A File is a locked *os.File. +// +// Closing the file releases the lock. +// +// If the program exits while a file is locked, the operating system releases +// the lock but may not do so promptly: callers must ensure that all locked +// files are closed before exiting. +type File struct { + osFile + closed bool + // cleanup panics when the file is no longer referenced and it has not been closed. + cleanup runtime.Cleanup +} + +// osFile embeds a *os.File while keeping the pointer itself unexported. +// (When we close a File, it must be the same file descriptor that we opened!) +type osFile struct { + *os.File +} + +// OpenFile is like os.OpenFile, but returns a locked file. +// If flag includes os.O_WRONLY or os.O_RDWR, the file is write-locked; +// otherwise, it is read-locked. +func OpenFile(name string, flag int, perm fs.FileMode) (*File, error) { + var ( + f = new(File) + err error + ) + f.osFile.File, err = openFile(name, flag, perm) + if err != nil { + return nil, err + } + + // Although the operating system will drop locks for open files when the go + // command exits, we want to hold locks for as little time as possible, and we + // especially don't want to leave a file locked after we're done with it. Our + // Close method is what releases the locks, so use a cleanup to report + // missing Close calls on a best-effort basis. + f.cleanup = runtime.AddCleanup(f, func(fileName string) { + panic(fmt.Sprintf("lockedfile.File %s became unreachable without a call to Close", fileName)) + }, f.Name()) + + return f, nil +} + +// Open is like os.Open, but returns a read-locked file. +func Open(name string) (*File, error) { + return OpenFile(name, os.O_RDONLY, 0) +} + +// Create is like os.Create, but returns a write-locked file. +func Create(name string) (*File, error) { + return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} + +// Edit creates the named file with mode 0666 (before umask), +// but does not truncate existing contents. +// +// If Edit succeeds, methods on the returned File can be used for I/O. +// The associated file descriptor has mode O_RDWR and the file is write-locked. +func Edit(name string) (*File, error) { + return OpenFile(name, os.O_RDWR|os.O_CREATE, 0666) +} + +// Close unlocks and closes the underlying file. +// +// Close may be called multiple times; all calls after the first will return a +// non-nil error. +func (f *File) Close() error { + if f.closed { + return &fs.PathError{ + Op: "close", + Path: f.Name(), + Err: fs.ErrClosed, + } + } + f.closed = true + + err := closeFile(f.osFile.File) + f.cleanup.Stop() + return err +} + +// Read opens the named file with a read-lock and returns its contents. +func Read(name string) ([]byte, error) { + f, err := Open(name) + if err != nil { + return nil, err + } + defer f.Close() + + return io.ReadAll(f) +} + +// Write opens the named file (creating it with the given permissions if needed), +// then write-locks it and overwrites it with the given content. +func Write(name string, content io.Reader, perm fs.FileMode) (err error) { + f, err := OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + + _, err = io.Copy(f, content) + if closeErr := f.Close(); err == nil { + err = closeErr + } + return err +} + +// Transform invokes t with the result of reading the named file, with its lock +// still held. +// +// If t returns a nil error, Transform then writes the returned contents back to +// the file, making a best effort to preserve existing contents on error. +// +// t must not modify the slice passed to it. +func Transform(name string, t func([]byte) ([]byte, error)) (err error) { + f, err := Edit(name) + if err != nil { + return err + } + defer f.Close() + + old, err := io.ReadAll(f) + if err != nil { + return err + } + + new, err := t(old) + if err != nil { + return err + } + + if len(new) > len(old) { + // The overall file size is increasing, so write the tail first: if we're + // about to run out of space on the disk, we would rather detect that + // failure before we have overwritten the original contents. + if _, err := f.WriteAt(new[len(old):], int64(len(old))); err != nil { + // Make a best effort to remove the incomplete tail. + f.Truncate(int64(len(old))) + return err + } + } + + // We're about to overwrite the old contents. In case of failure, make a best + // effort to roll back before we close the file. + defer func() { + if err != nil { + if _, err := f.WriteAt(old, 0); err == nil { + f.Truncate(int64(len(old))) + } + } + }() + + if len(new) >= len(old) { + if _, err := f.WriteAt(new[:len(old)], 0); err != nil { + return err + } + } else { + if _, err := f.WriteAt(new, 0); err != nil { + return err + } + // The overall file size is decreasing, so shrink the file to its final size + // after writing. We do this after writing (instead of before) so that if + // the write fails, enough filesystem space will likely still be reserved + // to contain the previous contents. + if err := f.Truncate(int64(len(new))); err != nil { + return err + } + } + + return nil +} diff --git a/internal/lockedfile/lockedfile_filelock.go b/internal/lockedfile/lockedfile_filelock.go new file mode 100644 index 0000000..9fc16a9 --- /dev/null +++ b/internal/lockedfile/lockedfile_filelock.go @@ -0,0 +1,65 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !plan9 + +package lockedfile + +import ( + "io/fs" + "os" + + "hakurei.app/internal/lockedfile/internal/filelock" +) + +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { + // On BSD systems, we could add the O_SHLOCK or O_EXLOCK flag to the OpenFile + // call instead of locking separately, but we have to support separate locking + // calls for Linux and Windows anyway, so it's simpler to use that approach + // consistently. + + f, err := os.OpenFile(name, flag&^os.O_TRUNC, perm) + if err != nil { + return nil, err + } + + switch flag & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR) { + case os.O_WRONLY, os.O_RDWR: + err = filelock.Lock(f) + default: + err = filelock.RLock(f) + } + if err != nil { + f.Close() + return nil, err + } + + if flag&os.O_TRUNC == os.O_TRUNC { + if err := f.Truncate(0); err != nil { + // The documentation for os.O_TRUNC says “if possible, truncate file when + // opened”, but doesn't define “possible” (golang.org/issue/28699). + // We'll treat regular files (and symlinks to regular files) as “possible” + // and ignore errors for the rest. + if fi, statErr := f.Stat(); statErr != nil || fi.Mode().IsRegular() { + filelock.Unlock(f) + f.Close() + return nil, err + } + } + } + + return f, nil +} + +func closeFile(f *os.File) error { + // Since locking syscalls operate on file descriptors, we must unlock the file + // while the descriptor is still valid — that is, before the file is closed — + // and avoid unlocking files that are already closed. + err := filelock.Unlock(f) + + if closeErr := f.Close(); err == nil { + err = closeErr + } + return err +} diff --git a/internal/lockedfile/lockedfile_plan9.go b/internal/lockedfile/lockedfile_plan9.go new file mode 100644 index 0000000..a2ce794 --- /dev/null +++ b/internal/lockedfile/lockedfile_plan9.go @@ -0,0 +1,94 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build plan9 + +package lockedfile + +import ( + "io/fs" + "math/rand" + "os" + "strings" + "time" +) + +// Opening an exclusive-use file returns an error. +// The expected error strings are: +// +// - "open/create -- file is locked" (cwfs, kfs) +// - "exclusive lock" (fossil) +// - "exclusive use file already open" (ramfs) +var lockedErrStrings = [...]string{ + "file is locked", + "exclusive lock", + "exclusive use file already open", +} + +// Even though plan9 doesn't support the Lock/RLock/Unlock functions to +// manipulate already-open files, IsLocked is still meaningful: os.OpenFile +// itself may return errors that indicate that a file with the ModeExclusive bit +// set is already open. +func isLocked(err error) bool { + s := err.Error() + + for _, frag := range lockedErrStrings { + if strings.Contains(s, frag) { + return true + } + } + + return false +} + +func openFile(name string, flag int, perm fs.FileMode) (*os.File, error) { + // Plan 9 uses a mode bit instead of explicit lock/unlock syscalls. + // + // Per http://man.cat-v.org/plan_9/5/stat: “Exclusive use files may be open + // for I/O by only one fid at a time across all clients of the server. If a + // second open is attempted, it draws an error.” + // + // So we can try to open a locked file, but if it fails we're on our own to + // figure out when it becomes available. We'll use exponential backoff with + // some jitter and an arbitrary limit of 500ms. + + // If the file was unpacked or created by some other program, it might not + // have the ModeExclusive bit set. Set it before we call OpenFile, so that we + // can be confident that a successful OpenFile implies exclusive use. + if fi, err := os.Stat(name); err == nil { + if fi.Mode()&fs.ModeExclusive == 0 { + if err := os.Chmod(name, fi.Mode()|fs.ModeExclusive); err != nil { + return nil, err + } + } + } else if !os.IsNotExist(err) { + return nil, err + } + + nextSleep := 1 * time.Millisecond + const maxSleep = 500 * time.Millisecond + for { + f, err := os.OpenFile(name, flag, perm|fs.ModeExclusive) + if err == nil { + return f, nil + } + + if !isLocked(err) { + return nil, err + } + + time.Sleep(nextSleep) + + nextSleep += nextSleep + if nextSleep > maxSleep { + nextSleep = maxSleep + } + // Apply 10% jitter to avoid synchronizing collisions. + nextSleep += time.Duration((0.1*rand.Float64() - 0.05) * float64(nextSleep)) + } +} + +func closeFile(f *os.File) error { + return f.Close() +} diff --git a/internal/lockedfile/lockedfile_test.go b/internal/lockedfile/lockedfile_test.go new file mode 100644 index 0000000..6182dd9 --- /dev/null +++ b/internal/lockedfile/lockedfile_test.go @@ -0,0 +1,263 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// js and wasip1 do not support inter-process file locking. +// +//go:build !js && !wasip1 + +package lockedfile_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "hakurei.app/container" + "hakurei.app/internal/lockedfile" + "hakurei.app/internal/lockedfile/internal/testexec" +) + +const ( + quiescent = 10 * time.Millisecond + probablyStillBlocked = 10 * time.Second +) + +func mustBlock(t *testing.T, desc string, f func()) (wait func(*testing.T)) { + t.Helper() + + done := make(chan struct{}) + go func() { + f() + close(done) + }() + + timer := time.NewTimer(quiescent) + defer timer.Stop() + select { + case <-done: + t.Fatalf("%s unexpectedly did not block", desc) + case <-timer.C: + } + + return func(t *testing.T) { + logTimer := time.NewTimer(quiescent) + defer logTimer.Stop() + + select { + case <-logTimer.C: + // We expect the operation to have unblocked by now, + // but maybe it's just slow. Write to the test log + // in case the test times out, but don't fail it. + t.Helper() + t.Logf("%s is unexpectedly still blocked after %v", desc, quiescent) + + // Wait for the operation to actually complete, no matter how long it + // takes. If the test has deadlocked, this will cause the test to time out + // and dump goroutines. + <-done + + case <-done: + } + } +} + +func TestMutexExcludes(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "lock") + mu := lockedfile.MutexAt(path) + t.Logf("mu := MutexAt(_)") + + unlock, err := mu.Lock() + if err != nil { + t.Fatalf("mu.Lock: %v", err) + } + t.Logf("unlock, _ := mu.Lock()") + + mu2 := lockedfile.MutexAt(mu.Path) + t.Logf("mu2 := MutexAt(mu.Path)") + + wait := mustBlock(t, "mu2.Lock()", func() { + unlock2, err := mu2.Lock() + if err != nil { + t.Errorf("mu2.Lock: %v", err) + return + } + t.Logf("unlock2, _ := mu2.Lock()") + t.Logf("unlock2()") + unlock2() + }) + + t.Logf("unlock()") + unlock() + wait(t) +} + +func TestReadWaitsForLock(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "timestamp.txt") + f, err := lockedfile.Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + defer f.Close() + + const ( + part1 = "part 1\n" + part2 = "part 2\n" + ) + _, err = f.WriteString(part1) + if err != nil { + t.Fatalf("WriteString: %v", err) + } + t.Logf("WriteString(%q) = ", part1) + + wait := mustBlock(t, "Read", func() { + b, err := lockedfile.Read(path) + if err != nil { + t.Errorf("Read: %v", err) + return + } + + const want = part1 + part2 + got := string(b) + if got == want { + t.Logf("Read(_) = %q", got) + } else { + t.Errorf("Read(_) = %q, _; want %q", got, want) + } + }) + + _, err = f.WriteString(part2) + if err != nil { + t.Errorf("WriteString: %v", err) + } else { + t.Logf("WriteString(%q) = ", part2) + } + f.Close() + + wait(t) +} + +func TestCanLockExistingFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "existing.txt") + if err := os.WriteFile(path, []byte("ok"), 0777); err != nil { + t.Fatalf("os.WriteFile: %v", err) + } + + f, err := lockedfile.Edit(path) + if err != nil { + t.Fatalf("first Edit: %v", err) + } + + wait := mustBlock(t, "Edit", func() { + other, err := lockedfile.Edit(path) + if err != nil { + t.Errorf("second Edit: %v", err) + } + other.Close() + }) + + f.Close() + wait(t) +} + +// TestSpuriousEDEADLK verifies that the spurious EDEADLK reported in +// https://golang.org/issue/32817 no longer occurs. +func TestSpuriousEDEADLK(t *testing.T) { + // P.1 locks file A. + // Q.3 locks file B. + // Q.3 blocks on file A. + // P.2 blocks on file B. (Spurious EDEADLK occurs here.) + // P.1 unlocks file A. + // Q.3 unblocks and locks file A. + // Q.3 unlocks files A and B. + // P.2 unblocks and locks file B. + // P.2 unlocks file B. + + dirVar := t.Name() + "DIR" + + if dir := os.Getenv(dirVar); dir != "" { + // Q.3 locks file B. + b, err := lockedfile.Edit(filepath.Join(dir, "B")) + if err != nil { + t.Fatal(err) + } + defer b.Close() + + if err := os.WriteFile(filepath.Join(dir, "locked"), []byte("ok"), 0666); err != nil { + t.Fatal(err) + } + + // Q.3 blocks on file A. + a, err := lockedfile.Edit(filepath.Join(dir, "A")) + // Q.3 unblocks and locks file A. + if err != nil { + t.Fatal(err) + } + defer a.Close() + + // Q.3 unlocks files A and B. + return + } + + dir := t.TempDir() + + // P.1 locks file A. + a, err := lockedfile.Edit(filepath.Join(dir, "A")) + if err != nil { + t.Fatal(err) + } + + cmd := testexec.CommandContext(t, t.Context(), container.MustExecutable(nil), "-test.run=^"+t.Name()+"$") + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", dirVar, dir)) + + qDone := make(chan struct{}) + waitQ := mustBlock(t, "Edit A and B in subprocess", func() { + out, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("%v:\n%s", err, out) + } + close(qDone) + }) + + // Wait until process Q has either failed or locked file B. + // Otherwise, P.2 might not block on file B as intended. +locked: + for { + if _, err := os.Stat(filepath.Join(dir, "locked")); !os.IsNotExist(err) { + break locked + } + timer := time.NewTimer(1 * time.Millisecond) + select { + case <-qDone: + timer.Stop() + break locked + case <-timer.C: + } + } + + waitP2 := mustBlock(t, "Edit B", func() { + // P.2 blocks on file B. (Spurious EDEADLK occurs here.) + b, err := lockedfile.Edit(filepath.Join(dir, "B")) + // P.2 unblocks and locks file B. + if err != nil { + t.Error(err) + return + } + // P.2 unlocks file B. + b.Close() + }) + + // P.1 unlocks file A. + a.Close() + + waitQ(t) + waitP2(t) +} diff --git a/internal/lockedfile/mutex.go b/internal/lockedfile/mutex.go new file mode 100644 index 0000000..180a36c --- /dev/null +++ b/internal/lockedfile/mutex.go @@ -0,0 +1,67 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lockedfile + +import ( + "fmt" + "os" + "sync" +) + +// A Mutex provides mutual exclusion within and across processes by locking a +// well-known file. Such a file generally guards some other part of the +// filesystem: for example, a Mutex file in a directory might guard access to +// the entire tree rooted in that directory. +// +// Mutex does not implement sync.Locker: unlike a sync.Mutex, a lockedfile.Mutex +// can fail to lock (e.g. if there is a permission error in the filesystem). +// +// Like a sync.Mutex, a Mutex may be included as a field of a larger struct but +// must not be copied after first use. The Path field must be set before first +// use and must not be change thereafter. +type Mutex struct { + Path string // The path to the well-known lock file. Must be non-empty. + mu sync.Mutex // A redundant mutex. The race detector doesn't know about file locking, so in tests we may need to lock something that it understands. +} + +// MutexAt returns a new Mutex with Path set to the given non-empty path. +func MutexAt(path string) *Mutex { + if path == "" { + panic("lockedfile.MutexAt: path must be non-empty") + } + return &Mutex{Path: path} +} + +func (mu *Mutex) String() string { + return fmt.Sprintf("lockedfile.Mutex(%s)", mu.Path) +} + +// Lock attempts to lock the Mutex. +// +// If successful, Lock returns a non-nil unlock function: it is provided as a +// return-value instead of a separate method to remind the caller to check the +// accompanying error. (See https://golang.org/issue/20803.) +func (mu *Mutex) Lock() (unlock func(), err error) { + if mu.Path == "" { + panic("lockedfile.Mutex: missing Path during Lock") + } + + // We could use either O_RDWR or O_WRONLY here. If we choose O_RDWR and the + // file at mu.Path is write-only, the call to OpenFile will fail with a + // permission error. That's actually what we want: if we add an RLock method + // in the future, it should call OpenFile with O_RDONLY and will require the + // files must be readable, so we should not let the caller make any + // assumptions about Mutex working with write-only files. + f, err := OpenFile(mu.Path, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return nil, err + } + mu.mu.Lock() + + return func() { + mu.mu.Unlock() + f.Close() + }, nil +} diff --git a/internal/lockedfile/transform_test.go b/internal/lockedfile/transform_test.go new file mode 100644 index 0000000..1394ada --- /dev/null +++ b/internal/lockedfile/transform_test.go @@ -0,0 +1,103 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// js and wasip1 do not support inter-process file locking. +// +//go:build !js && !wasip1 + +package lockedfile_test + +import ( + "bytes" + "encoding/binary" + "math/rand" + "path/filepath" + "testing" + "time" + + "hakurei.app/internal/lockedfile" +) + +func isPowerOf2(x int) bool { + return x > 0 && x&(x-1) == 0 +} + +func roundDownToPowerOf2(x int) int { + if x <= 0 { + panic("nonpositive x") + } + bit := 1 + for x != bit { + x = x &^ bit + bit <<= 1 + } + return x +} + +func TestTransform(t *testing.T) { + path := filepath.Join(t.TempDir(), "blob.bin") + + const maxChunkWords = 8 << 10 + buf := make([]byte, 2*maxChunkWords*8) + for i := uint64(0); i < 2*maxChunkWords; i++ { + binary.LittleEndian.PutUint64(buf[i*8:], i) + } + if err := lockedfile.Write(path, bytes.NewReader(buf[:8]), 0666); err != nil { + t.Fatal(err) + } + + var attempts int64 = 128 + if !testing.Short() { + attempts *= 16 + } + const parallel = 32 + + var sem = make(chan bool, parallel) + + for n := attempts; n > 0; n-- { + sem <- true + go func() { + defer func() { <-sem }() + + time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) + chunkWords := roundDownToPowerOf2(rand.Intn(maxChunkWords) + 1) + offset := rand.Intn(chunkWords) + + err := lockedfile.Transform(path, func(data []byte) (chunk []byte, err error) { + chunk = buf[offset*8 : (offset+chunkWords)*8] + + if len(data)&^7 != len(data) { + t.Errorf("read %d bytes, but each write is an integer multiple of 8 bytes", len(data)) + return chunk, nil + } + + words := len(data) / 8 + if !isPowerOf2(words) { + t.Errorf("read %d 8-byte words, but each write is a power-of-2 number of words", words) + return chunk, nil + } + + u := binary.LittleEndian.Uint64(data) + for i := 1; i < words; i++ { + next := binary.LittleEndian.Uint64(data[i*8:]) + if next != u+1 { + t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i) + return chunk, nil + } + u = next + } + + return chunk, nil + }) + + if err != nil { + t.Errorf("unexpected error from Transform: %v", err) + } + }() + } + + for n := parallel; n > 0; n-- { + sem <- true + } +}