From 1b3902df7814e0f5e19d88acdc4588f6da1c895f Mon Sep 17 00:00:00 2001 From: Ophestra Date: Sat, 23 Aug 2025 11:06:19 +0900 Subject: [PATCH] container/dispatcher: instrument each goroutine individually Scheduler nondeterminism cannot be accounted for, so do this instead. There should not be any performance penalty as these calls are optimised out for direct. Signed-off-by: Ophestra --- container/dispatcher.go | 7 +++ container/dispatcher_test.go | 55 ++++++++++++++++------- container/init.go | 4 +- container/mount_test.go | 84 ++++++++++++++++++------------------ 4 files changed, 91 insertions(+), 59 deletions(-) diff --git a/container/dispatcher.go b/container/dispatcher.go index 78648a7..c7a8bad 100644 --- a/container/dispatcher.go +++ b/container/dispatcher.go @@ -22,6 +22,11 @@ type osFile interface { // syscallDispatcher provides methods that make state-dependent system calls as part of their behaviour. type syscallDispatcher interface { + // new returns a new instance of syscallDispatcher for use in another goroutine. + // A syscallDispatcher must never be used in any goroutine other than the one owning it, + // just synchronising access is not enough, as this is for test instrumentation. + new() syscallDispatcher + // lockOSThread provides [runtime.LockOSThread]. lockOSThread() @@ -140,6 +145,8 @@ type syscallDispatcher interface { // direct implements syscallDispatcher on the current kernel. type direct struct{} +func (k direct) new() syscallDispatcher { return k } + func (direct) lockOSThread() { runtime.LockOSThread() } func (direct) setPtracer(pid uintptr) error { return SetPtracer(pid) } diff --git a/container/dispatcher_test.go b/container/dispatcher_test.go index 012673a..bfb38cd 100644 --- a/container/dispatcher_test.go +++ b/container/dispatcher_test.go @@ -106,7 +106,7 @@ func checkOpMeta(t *testing.T, testCases []opMetaTestCase) { type simpleTestCase struct { name string f func(k syscallDispatcher) error - want []kexpect + want [][]kexpect wantErr error } @@ -117,9 +117,9 @@ func checkSimple(t *testing.T, fname string, testCases []simpleTestCase) { if err := tc.f(k); !errors.Is(err, tc.wantErr) { t.Errorf("%s: error = %v, want %v", fname, err, tc.wantErr) } - if len(k.want) != k.pos { - t.Errorf("%s: %d calls, want %d", fname, k.pos, len(k.want)) - } + k.handleIncomplete(func(k *kstub) { + t.Errorf("%s: %d calls, want %d (track %d)", fname, k.pos, len(k.want[k.track]), k.track) + }) }) } } @@ -141,7 +141,7 @@ func checkOpBehaviour(t *testing.T, testCases []opBehaviourTestCase) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { state := &setupState{Params: tc.params} - k := &kstub{t: t, want: slices.Concat(tc.early, []kexpect{{name: "\x00"}}, tc.apply)} + k := &kstub{t: t, want: [][]kexpect{slices.Concat(tc.early, []kexpect{{name: "\x00"}}, tc.apply)}} errEarly := tc.op.early(state, k) k.expect("\x00") if !errors.Is(errEarly, tc.wantErrEarly) { @@ -156,14 +156,14 @@ func checkOpBehaviour(t *testing.T, testCases []opBehaviourTestCase) { } out: - if len(k.want) != k.pos { + k.handleIncomplete(func(k *kstub) { count := k.pos - 1 // separator if count < len(tc.early) { t.Errorf("early: %d calls, want %d", count, len(tc.early)) } else { t.Errorf("apply: %d calls, want %d", count-len(tc.early), len(tc.apply)) } - } + }) }) } }) @@ -263,17 +263,33 @@ func (k *kexpect) error(ok ...bool) error { } type kstub struct { - t *testing.T - want []kexpect - pos int + t *testing.T + + want [][]kexpect + // pos is the current position in want[track]. + pos int + // track is the current active want. + track int + // sub stores addresses of kstub created by new. + sub []*kstub +} + +// handleIncomplete calls f on an incomplete k and all its descendants. +func (k *kstub) handleIncomplete(f func(k *kstub)) { + if k.want != nil && len(k.want[k.track]) != k.pos { + f(k) + } + for _, sk := range k.sub { + sk.handleIncomplete(f) + } } // expect checks name and returns the current kexpect and advances pos. func (k *kstub) expect(name string) (expect *kexpect) { - if len(k.want) == k.pos { + if len(k.want[k.track]) == k.pos { k.t.Fatal("expect: want too short") } - expect = &k.want[k.pos] + expect = &k.want[k.track][k.pos] if name != expect.name { if expect.name == "\x00" { k.t.Fatalf("expect: func = %s, separator overrun", name) @@ -292,7 +308,7 @@ func checkArg[T comparable](k *kstub, arg string, got T, n int) bool { if k.pos == 0 { panic("invalid call to checkArg") } - expect := k.want[k.pos-1] + expect := k.want[k.track][k.pos-1] want, ok := expect.args[n].(T) if !ok || got != want { k.t.Errorf("%s: %s = %#v, want %#v (%d)", expect.name, arg, got, want, k.pos-1) @@ -306,7 +322,7 @@ func checkArgReflect(k *kstub, arg string, got any, n int) bool { if k.pos == 0 { panic("invalid call to checkArgReflect") } - expect := k.want[k.pos-1] + expect := k.want[k.track][k.pos-1] want := expect.args[n] if !reflect.DeepEqual(got, want) { k.t.Errorf("%s: %s = %#v, want %#v (%d)", expect.name, arg, got, want, k.pos-1) @@ -315,6 +331,15 @@ func checkArgReflect(k *kstub, arg string, got any, n int) bool { return true } +func (k *kstub) new() syscallDispatcher { + k.expect("new") + if len(k.want) <= k.track+1 { + k.t.Fatalf("new: track overrun") + } + k.sub = append(k.sub, &kstub{t: k.t, want: k.want, track: k.track + 1}) + return k.sub[len(k.sub)-1] +} + func (k *kstub) lockOSThread() { k.expect("lockOSThread") } func (k *kstub) setPtracer(pid uintptr) error { @@ -328,7 +353,7 @@ func (k *kstub) setDumpable(dumpable uintptr) error { } func (k *kstub) setNoNewPrivs() error { return k.expect("setNoNewPrivs").err } -func (k *kstub) lastcap() uintptr { return k.expect("setNoNewPrivs").ret.(uintptr) } +func (k *kstub) lastcap() uintptr { return k.expect("lastcap").ret.(uintptr) } func (k *kstub) capset(hdrp *capHeader, datap *[2]capData) error { return k.expect("capset").error( diff --git a/container/init.go b/container/init.go index 813b4ac..39adefb 100644 --- a/container/init.go +++ b/container/init.go @@ -333,7 +333,7 @@ func initEntrypoint(k syscallDispatcher, prepareLogger func(prefix string), setV info := make(chan winfo, 1) done := make(chan struct{}) - go func() { + go func(k syscallDispatcher) { var ( err error wpid = -2 @@ -360,7 +360,7 @@ func initEntrypoint(k syscallDispatcher, prepareLogger func(prefix string), setV } close(done) - }() + }(k.new()) // handle signals to dump withheld messages sig := make(chan os.Signal, 2) diff --git a/container/mount_test.go b/container/mount_test.go index 0c26f6f..2b5b1c4 100644 --- a/container/mount_test.go +++ b/container/mount_test.go @@ -12,26 +12,26 @@ func TestBindMount(t *testing.T) { checkSimple(t, "bindMount", []simpleTestCase{ {"mount", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).bindMount("/host/nix", "/sysroot/nix", syscall.MS_RDONLY, true) - }, []kexpect{ + }, [][]kexpect{{ {"verbosef", expectArgs{"resolved %q flags %#x", []any{"/sysroot/nix", uintptr(1)}}, nil, nil}, {"mount", expectArgs{"/host/nix", "/sysroot/nix", "", uintptr(0x9000), ""}, nil, errUnique}, - }, wrapErrSuffix(errUnique, `cannot mount "/host/nix" on "/sysroot/nix":`)}, + }}, wrapErrSuffix(errUnique, `cannot mount "/host/nix" on "/sysroot/nix":`)}, {"success ne", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).bindMount("/host/nix", "/sysroot/.host-nix", syscall.MS_RDONLY, false) - }, []kexpect{ + }, [][]kexpect{{ {"verbosef", expectArgs{"resolved %q on %q flags %#x", []any{"/host/nix", "/sysroot/.host-nix", uintptr(1)}}, nil, nil}, {"mount", expectArgs{"/host/nix", "/sysroot/.host-nix", "", uintptr(0x9000), ""}, nil, nil}, {"remount", expectArgs{"/sysroot/.host-nix", uintptr(1)}, nil, nil}, - }, nil}, + }}, nil}, {"success", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).bindMount("/host/nix", "/sysroot/nix", syscall.MS_RDONLY, true) - }, []kexpect{ + }, [][]kexpect{{ {"verbosef", expectArgs{"resolved %q flags %#x", []any{"/sysroot/nix", uintptr(1)}}, nil, nil}, {"mount", expectArgs{"/host/nix", "/sysroot/nix", "", uintptr(0x9000), ""}, nil, nil}, {"remount", expectArgs{"/sysroot/nix", uintptr(1)}, nil, nil}, - }, nil}, + }}, nil}, }) } @@ -81,69 +81,69 @@ func TestRemount(t *testing.T) { checkSimple(t, "remount", []simpleTestCase{ {"evalSymlinks", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", errUnique}, - }, wrapErrSelf(errUnique)}, + }}, wrapErrSelf(errUnique)}, {"open", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, errUnique}, - }, wrapErrSuffix(errUnique, `cannot open "/sysroot/nix":`)}, + }}, wrapErrSuffix(errUnique, `cannot open "/sysroot/nix":`)}, {"readlink", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", errUnique}, - }, wrapErrSelf(errUnique)}, + }}, wrapErrSelf(errUnique)}, {"close", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, {"close", expectArgs{0xdeadbeef}, nil, errUnique}, - }, wrapErrSuffix(errUnique, `cannot close "/sysroot/nix":`)}, + }}, wrapErrSuffix(errUnique, `cannot close "/sysroot/nix":`)}, {"mountinfo stale", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/.hakurei", nil}, {"verbosef", expectArgs{"target resolves to %q", []any{"/sysroot/.hakurei"}}, nil, nil}, {"open", expectArgs{"/sysroot/.hakurei", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/.hakurei", nil}, {"close", expectArgs{0xdeadbeef}, nil, nil}, {"openNew", expectArgs{"/host/proc/self/mountinfo"}, newConstFile(sampleMountinfoNix), nil}, - }, msg.WrapErr(syscall.ESTALE, `mount point "/sysroot/.hakurei" never appeared in mountinfo`)}, + }}, msg.WrapErr(syscall.ESTALE, `mount point "/sysroot/.hakurei" never appeared in mountinfo`)}, {"mountinfo", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, {"close", expectArgs{0xdeadbeef}, nil, nil}, {"openNew", expectArgs{"/host/proc/self/mountinfo"}, newConstFile("\x00"), nil}, - }, wrapErrSuffix(vfs.ErrMountInfoFields, `cannot parse mountinfo:`)}, + }}, wrapErrSuffix(vfs.ErrMountInfoFields, `cannot parse mountinfo:`)}, {"mount", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, {"close", expectArgs{0xdeadbeef}, nil, nil}, {"openNew", expectArgs{"/host/proc/self/mountinfo"}, newConstFile(sampleMountinfoNix), nil}, {"mount", expectArgs{"none", "/sysroot/nix", "", uintptr(0x209027), ""}, nil, errUnique}, - }, wrapErrSuffix(errUnique, `cannot remount "/sysroot/nix":`)}, + }}, wrapErrSuffix(errUnique, `cannot remount "/sysroot/nix":`)}, {"mount propagate", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, @@ -151,22 +151,22 @@ func TestRemount(t *testing.T) { {"openNew", expectArgs{"/host/proc/self/mountinfo"}, newConstFile(sampleMountinfoNix), nil}, {"mount", expectArgs{"none", "/sysroot/nix", "", uintptr(0x209027), ""}, nil, nil}, {"mount", expectArgs{"none", "/sysroot/nix/.ro-store", "", uintptr(0x209027), ""}, nil, errUnique}, - }, wrapErrSuffix(errUnique, `cannot propagate flags to "/sysroot/nix/.ro-store":`)}, + }}, wrapErrSuffix(errUnique, `cannot propagate flags to "/sysroot/nix/.ro-store":`)}, {"success toplevel", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/bin", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/bin"}, "/sysroot/bin", nil}, {"open", expectArgs{"/sysroot/bin", 0x280000, uint32(0)}, 0xbabe, nil}, {"readlink", expectArgs{"/host/proc/self/fd/47806"}, "/sysroot/bin", nil}, {"close", expectArgs{0xbabe}, nil, nil}, {"openNew", expectArgs{"/host/proc/self/mountinfo"}, newConstFile(sampleMountinfoNix), nil}, {"mount", expectArgs{"none", "/sysroot/bin", "", uintptr(0x209027), ""}, nil, nil}, - }, nil}, + }}, nil}, {"success EACCES", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, @@ -175,22 +175,22 @@ func TestRemount(t *testing.T) { {"mount", expectArgs{"none", "/sysroot/nix", "", uintptr(0x209027), ""}, nil, nil}, {"mount", expectArgs{"none", "/sysroot/nix/.ro-store", "", uintptr(0x209027), ""}, nil, syscall.EACCES}, {"mount", expectArgs{"none", "/sysroot/nix/store", "", uintptr(0x209027), ""}, nil, nil}, - }, nil}, + }}, nil}, {"success no propagate", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, {"close", expectArgs{0xdeadbeef}, nil, nil}, {"openNew", expectArgs{"/host/proc/self/mountinfo"}, newConstFile(sampleMountinfoNix), nil}, {"mount", expectArgs{"none", "/sysroot/nix", "", uintptr(0x209027), ""}, nil, nil}, - }, nil}, + }}, nil}, {"success case sensitive", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/nix"}, "/sysroot/nix", nil}, {"open", expectArgs{"/sysroot/nix", 0x280000, uint32(0)}, 0xdeadbeef, nil}, {"readlink", expectArgs{"/host/proc/self/fd/3735928559"}, "/sysroot/nix", nil}, @@ -199,11 +199,11 @@ func TestRemount(t *testing.T) { {"mount", expectArgs{"none", "/sysroot/nix", "", uintptr(0x209027), ""}, nil, nil}, {"mount", expectArgs{"none", "/sysroot/nix/.ro-store", "", uintptr(0x209027), ""}, nil, nil}, {"mount", expectArgs{"none", "/sysroot/nix/store", "", uintptr(0x209027), ""}, nil, nil}, - }, nil}, + }}, nil}, {"success", func(k syscallDispatcher) error { return newProcPaths(k, hostPath).remount("/sysroot/.nix", syscall.MS_REC|syscall.MS_RDONLY|syscall.MS_NODEV) - }, []kexpect{ + }, [][]kexpect{{ {"evalSymlinks", expectArgs{"/sysroot/.nix"}, "/sysroot/NIX", nil}, {"verbosef", expectArgs{"target resolves to %q", []any{"/sysroot/NIX"}}, nil, nil}, {"open", expectArgs{"/sysroot/NIX", 0x280000, uint32(0)}, 0xdeadbeef, nil}, @@ -213,7 +213,7 @@ func TestRemount(t *testing.T) { {"mount", expectArgs{"none", "/sysroot/nix", "", uintptr(0x209027), ""}, nil, nil}, {"mount", expectArgs{"none", "/sysroot/nix/.ro-store", "", uintptr(0x209027), ""}, nil, nil}, {"mount", expectArgs{"none", "/sysroot/nix/store", "", uintptr(0x209027), ""}, nil, nil}, - }, nil}, + }}, nil}, }) } @@ -221,9 +221,9 @@ func TestRemountWithFlags(t *testing.T) { checkSimple(t, "remountWithFlags", []simpleTestCase{ {"noop unmatched", func(k syscallDispatcher) error { return remountWithFlags(k, &vfs.MountInfoNode{MountInfoEntry: &vfs.MountInfoEntry{VfsOptstr: "rw,relatime,cat"}}, 0) - }, []kexpect{ + }, [][]kexpect{{ {"verbosef", expectArgs{"unmatched vfs options: %q", []any{[]string{"cat"}}}, nil, nil}, - }, nil}, + }}, nil}, {"noop", func(k syscallDispatcher) error { return remountWithFlags(k, &vfs.MountInfoNode{MountInfoEntry: &vfs.MountInfoEntry{VfsOptstr: "rw,relatime"}}, 0) @@ -231,9 +231,9 @@ func TestRemountWithFlags(t *testing.T) { {"success", func(k syscallDispatcher) error { return remountWithFlags(k, &vfs.MountInfoNode{MountInfoEntry: &vfs.MountInfoEntry{VfsOptstr: "rw,relatime"}}, syscall.MS_RDONLY) - }, []kexpect{ + }, [][]kexpect{{ {"mount", expectArgs{"none", "", "", uintptr(0x209021), ""}, nil, nil}, - }, nil}, + }}, nil}, }) } @@ -241,23 +241,23 @@ func TestMountTmpfs(t *testing.T) { checkSimple(t, "mountTmpfs", []simpleTestCase{ {"mkdirAll", func(k syscallDispatcher) error { return mountTmpfs(k, "ephemeral", "/sysroot/run/user/1000", 0, 1<<10, 0700) - }, []kexpect{ + }, [][]kexpect{{ {"mkdirAll", expectArgs{"/sysroot/run/user/1000", os.FileMode(0700)}, nil, errUnique}, - }, wrapErrSelf(errUnique)}, + }}, wrapErrSelf(errUnique)}, {"success no size", func(k syscallDispatcher) error { return mountTmpfs(k, "ephemeral", "/sysroot/run/user/1000", 0, 0, 0710) - }, []kexpect{ + }, [][]kexpect{{ {"mkdirAll", expectArgs{"/sysroot/run/user/1000", os.FileMode(0750)}, nil, nil}, {"mount", expectArgs{"ephemeral", "/sysroot/run/user/1000", "tmpfs", uintptr(0), "mode=0710"}, nil, nil}, - }, nil}, + }}, nil}, {"success", func(k syscallDispatcher) error { return mountTmpfs(k, "ephemeral", "/sysroot/run/user/1000", 0, 1<<10, 0700) - }, []kexpect{ + }, [][]kexpect{{ {"mkdirAll", expectArgs{"/sysroot/run/user/1000", os.FileMode(0700)}, nil, nil}, {"mount", expectArgs{"ephemeral", "/sysroot/run/user/1000", "tmpfs", uintptr(0), "mode=0700,size=1024"}, nil, nil}, - }, nil}, + }}, nil}, }) }