internal/pkg: implement exec artifact

This runs a program in a container environment. Artifacts can be made available to the container, they are cured concurrently and mounted in order.

Signed-off-by: Ophestra <cat@gensokyo.uk>

.gitignore

@@ -27,6 +27,7 @@ go.work.sum
 
 # go generate
 /cmd/hakurei/LICENSE
+/internal/pkg/testdata/testtool
 
 # release
 /dist/hakurei-*

internal/pkg/dir_test.go

@@ -262,6 +262,44 @@ func TestFlatten(t *testing.T) {
 			{Mode: fs.ModeDir | 0700, Path: "temp"},
 			{Mode: fs.ModeDir | 0700, Path: "work"},
 		}, pkg.MustDecode("YZUoGdMwmfW5sQWto9hQgMKah648rHKck8Ds_GGnqgCBpTAiZKOefpHCKnvktfYh")},
+
+		{"testtool", fstest.MapFS{
+			".": {Mode: fs.ModeDir | 0500},
+
+			"check": {Mode: 0400, Data: []byte{0}},
+		}, []pkg.FlatEntry{
+			{Mode: fs.ModeDir | 0500, Path: "."},
+
+			{Mode: 0400, Path: "check", Data: []byte{0}},
+		}, pkg.MustDecode("GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9")},
+
+		{"sample exec container", fstest.MapFS{
+			".":        {Mode: fs.ModeDir | 0700},
+			"checksum": {Mode: fs.ModeDir | 0700},
+			"checksum/GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9":       {Mode: fs.ModeDir | 0500},
+			"checksum/GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9/check": {Mode: 0400, Data: []byte{0x0}},
+			"checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU":       {Mode: fs.ModeDir | 0500},
+			"checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb":       {Mode: 0400, Data: []byte{}},
+			"identifier": {Mode: fs.ModeDir | 0700},
+			"identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+			"identifier/nfeISfLeFDr1k-g3hpE1oZ440kTqDdfF8TDpoLdbTPqaMMIl95oiqcvqjRkMjubA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			"identifier/oHuqV7p0v1Vd8IdAzjyYM8sfCS0P2LR5tfv5cb6Gbf2ZWUm8Ec-7hYPJ_qr183m7": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9")},
+			"temp": {Mode: fs.ModeDir | 0700},
+			"work": {Mode: fs.ModeDir | 0700},
+		}, []pkg.FlatEntry{
+			{Mode: fs.ModeDir | 0700, Path: "."},
+			{Mode: fs.ModeDir | 0700, Path: "checksum"},
+			{Mode: fs.ModeDir | 0500, Path: "checksum/GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9"},
+			{Mode: 0400, Path: "checksum/GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9/check", Data: []byte{0x0}},
+			{Mode: fs.ModeDir | 0500, Path: "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU"},
+			{Mode: 0400, Path: "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb", Data: []byte{}},
+			{Mode: fs.ModeDir | 0700, Path: "identifier"},
+			{Mode: fs.ModeSymlink | 0777, Path: "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+			{Mode: fs.ModeSymlink | 0777, Path: "identifier/nfeISfLeFDr1k-g3hpE1oZ440kTqDdfF8TDpoLdbTPqaMMIl95oiqcvqjRkMjubA", Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+			{Mode: fs.ModeSymlink | 0777, Path: "identifier/oHuqV7p0v1Vd8IdAzjyYM8sfCS0P2LR5tfv5cb6Gbf2ZWUm8Ec-7hYPJ_qr183m7", Data: []byte("../checksum/GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9")},
+			{Mode: fs.ModeDir | 0700, Path: "temp"},
+			{Mode: fs.ModeDir | 0700, Path: "work"},
+		}, pkg.MustDecode("7PoPpWLjFPXIymbuIYLZAzOpCYr-2PN4CZ11jFdO-mDlnZNgFO3JyOtK8HW8Jxvm")},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

internal/pkg/exec.go

@@ -0,0 +1,223 @@
+package pkg
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"runtime"
+
+	"hakurei.app/container"
+	"hakurei.app/container/check"
+	"hakurei.app/container/fhs"
+	"hakurei.app/container/std"
+	"hakurei.app/message"
+)
+
+// ExecContainerPath is an [Artifact] and the [check.Absolute] pathname to make
+// it available under in the container.
+type ExecContainerPath struct {
+	P *check.Absolute
+	A Artifact
+}
+
+// MustPath returns [ExecContainerPath] for pathname and [Artifact] and panics
+// if pathname is not absolute.
+func MustPath(pathname string, a Artifact) ExecContainerPath {
+	return ExecContainerPath{check.MustAbs(pathname), a}
+}
+
+// An execArtifact is an [Artifact] that produces output by running a program
+// part of another [Artifact] in a [container] to produce its output.
+type execArtifact struct {
+	// Caller-supplied context.
+	ctx context.Context
+	// Caller-supplied inner read-only bind mounts.
+	paths []ExecContainerPath
+	// Caller-supplied logging facility, passed through to [container] and used
+	// internally to produce verbose output.
+	msg message.Msg
+
+	// Number of [Artifact] to concurrently cure. A value of 0 or lower is
+	// equivalent to the value returned by [runtime.NumCPU].
+	cures int
+
+	// Passed through to [container.Params].
+	dir *check.Absolute
+	// Passed through to [container.Params].
+	env []string
+	// Passed through to [container.Params].
+	path *check.Absolute
+	// Passed through to [container.Params].
+	args []string
+}
+
+// NewExec returns a new [Artifact] bounded by ctx, it cures all [Artifact]
+// in paths at the specified maximum concurrent cures limit. Specified paths are
+// bind mounted read-only in the specified order in the resulting container.
+// A private instance of /proc and /dev is made available to the container.
+//
+// The working and temporary directories are both created and mounted writable
+// on /work and /tmp respectively.
+//
+// A cures value of 0 or lower is equivalent to the value returned by
+// [runtime.NumCPU].
+func NewExec(
+	ctx context.Context,
+	msg message.Msg,
+	cures int,
+
+	dir *check.Absolute,
+	env []string,
+	path *check.Absolute,
+	args []string,
+
+	paths ...ExecContainerPath,
+) Artifact {
+	return &execArtifact{ctx, paths, msg, cures, dir, env, path, args}
+}
+
+// Kind returns the hardcoded [Kind] constant.
+func (a *execArtifact) Kind() Kind { return KindExec }
+
+// Params returns paths, executable pathname and args concatenated together.
+func (a *execArtifact) Params() []byte {
+	var buf bytes.Buffer
+	for _, p := range a.paths {
+		buf.WriteString(p.P.String())
+		id := Ident(p.A)
+		buf.Write(id[:])
+	}
+	buf.WriteByte(0)
+	buf.WriteString(a.dir.String())
+	buf.WriteByte(0)
+	for _, e := range a.env {
+		buf.WriteString(e)
+	}
+	buf.WriteByte(0)
+	buf.WriteString(a.path.String())
+	buf.WriteByte(0)
+	for _, arg := range a.args {
+		buf.WriteString(arg)
+	}
+	return buf.Bytes()
+}
+
+// Dependencies returns a slice of all artifacts collected from caller-supplied
+// [ExecContainerPath].
+func (a *execArtifact) Dependencies() []Artifact {
+	artifacts := make([]Artifact, len(a.paths))
+	for i, p := range a.paths {
+		artifacts[i] = p.A
+	}
+	return artifacts
+}
+
+// Cure cures the [Artifact] by curing all its dependencies then running the
+// container described by the caller.
+func (a *execArtifact) Cure(c *CureContext) (err error) {
+	cures := a.cures
+	if cures < 1 {
+		cures = runtime.NumCPU()
+	}
+
+	paths := make([][2]*check.Absolute, len(a.paths))
+	for i, p := range a.paths {
+		paths[i][1] = p.P
+	}
+
+	if len(paths) > 0 {
+		type cureArtifact struct {
+			// Index of pending Artifact in paths.
+			index int
+			// Pending artifact.
+			a Artifact
+		}
+		ac := make(chan cureArtifact, len(paths))
+		for i, p := range a.paths {
+			ac <- cureArtifact{i, p.A}
+		}
+
+		type cureRes struct {
+			// Index of result in paths.
+			index int
+			// Cured pathname.
+			pathname *check.Absolute
+			// Error returned by c.
+			err error
+		}
+		res := make(chan cureRes)
+
+		for i := 0; i < cures; i++ {
+			go func() {
+				for d := range ac {
+					// computing and encoding identifier is expensive
+					if a.msg.IsVerbose() {
+						a.msg.Verbosef("curing %s...", Encode(Ident(d.a)))
+					}
+
+					var cr cureRes
+					cr.index = d.index
+					cr.pathname, _, cr.err = c.Cure(d.a)
+					res <- cr
+				}
+			}()
+		}
+
+		var count int
+		errs := make([]error, 0, len(paths))
+		for cr := range res {
+			count++
+
+			if cr.err != nil {
+				errs = append(errs, cr.err)
+			} else {
+				paths[cr.index][0] = cr.pathname
+			}
+
+			if count == len(paths) {
+				break
+			}
+		}
+		close(ac)
+		if err = errors.Join(errs...); err != nil {
+			return
+		}
+	}
+
+	ctx, cancel := context.WithCancel(a.ctx)
+	defer cancel()
+
+	z := container.New(ctx, a.msg)
+	z.ForwardCancel = true
+	z.SeccompPresets |= std.PresetStrict
+	z.ParentPerm = 0700
+	z.Hostname = "cure"
+	z.Uid, z.Gid = (1<<10)-1, (1<<10)-1
+	if a.msg.IsVerbose() {
+		z.Stdin, z.Stdout, z.Stderr = os.Stdin, os.Stdout, os.Stderr
+	}
+
+	z.Dir, z.Env, z.Path, z.Args = a.dir, a.env, a.path, a.args
+	z.Grow(len(paths) + 4)
+	for _, b := range paths {
+		z.Bind(b[0], b[1], 0)
+	}
+	z.Bind(
+		c.GetWorkDir(),
+		fhs.AbsRoot.Append("work"),
+		std.BindWritable|std.BindEnsure,
+	).Bind(
+		c.GetTempDir(),
+		fhs.AbsTmp,
+		std.BindWritable|std.BindEnsure,
+	).Proc(fhs.AbsProc).Dev(fhs.AbsDev, true)
+
+	if err = z.Start(); err != nil {
+		return
+	}
+	if err = z.Serve(); err != nil {
+		return
+	}
+	return z.Wait()
+}

internal/pkg/exec_test.go

@@ -0,0 +1,138 @@
+package pkg_test
+
+//go:generate go build -tags testtool -ldflags "-extldflags='-static'" -o testdata/testtool ./testdata
+
+import (
+	_ "embed"
+	"errors"
+	"log"
+	"os"
+	"os/exec"
+	"testing"
+
+	"hakurei.app/container/check"
+	"hakurei.app/container/stub"
+	"hakurei.app/hst"
+	"hakurei.app/internal/pkg"
+	"hakurei.app/message"
+)
+
+// testtoolBin is the container test tool binary made available to the
+// execArtifact for testing its curing environment.
+//
+//go:embed testdata/testtool
+var testtoolBin []byte
+
+func TestExec(t *testing.T) {
+	t.Parallel()
+
+	checkWithCache(t, []cacheTestCase{
+		{"container", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+			c.SetStrict(true)
+
+			// this is built during go:generate and is not deterministic
+			testtool := overrideIdent{pkg.ID{0xfe, 0xff}, stubArtifact{
+				kind: pkg.KindTar,
+				cure: func(c *pkg.CureContext) error {
+					work := c.GetWorkDir()
+					if err := os.MkdirAll(
+						work.Append("bin").String(),
+						0700,
+					); err != nil {
+						return err
+					}
+					return os.WriteFile(c.GetWorkDir().Append(
+						"bin",
+						"testtool",
+					).String(), testtoolBin, 0500)
+				},
+			}}
+
+			msg := message.New(log.New(os.Stderr, "container: ", 0))
+			msg.SwapVerbose(testing.Verbose())
+
+			cureMany(t, c, []cureStep{
+				{"container", pkg.NewExec(
+					t.Context(),
+					msg,
+					0,
+					check.MustAbs("/work"),
+					[]string{"HAKUREI_TEST=1"},
+					check.MustAbs("/opt/bin/testtool"),
+					[]string{"testtool"},
+
+					pkg.MustPath("/file", newStubFile(
+						pkg.KindHTTPGet,
+						pkg.ID{0xfe, 0},
+						nil,
+						nil, nil,
+					)),
+					pkg.MustPath("/.hakurei", stubArtifact{
+						kind:   pkg.KindTar,
+						params: []byte("empty directory"),
+						cure: func(c *pkg.CureContext) error {
+							return os.MkdirAll(c.GetWorkDir().String(), 0700)
+						},
+					}),
+					pkg.MustPath("/opt", testtool),
+				), ignorePathname, pkg.MustDecode(
+					"GPa4aBakdSJd7Tz7LYj_VJFoojzyZinmVcG3k6M5xI6CZ821J5sXLhLDDuS47gi9",
+				), nil},
+
+				{"error passthrough", pkg.NewExec(
+					t.Context(),
+					msg,
+					0,
+					check.MustAbs("/work"),
+					[]string{"HAKUREI_TEST=1"},
+					check.MustAbs("/opt/bin/testtool"),
+					[]string{"testtool"},
+
+					pkg.MustPath("/proc/nonexistent", stubArtifact{
+						kind:   pkg.KindTar,
+						params: []byte("doomed artifact"),
+						cure: func(c *pkg.CureContext) error {
+							return stub.UniqueError(0xcafe)
+						},
+					}),
+				), nil, pkg.Checksum{}, errors.Join(stub.UniqueError(0xcafe))},
+			})
+
+			// check init failure passthrough
+			var exitError *exec.ExitError
+			if _, _, err := c.Cure(pkg.NewExec(
+				t.Context(),
+				msg,
+				0,
+				check.MustAbs("/work"),
+				nil,
+				check.MustAbs("/opt/bin/testtool"),
+				[]string{"testtool"},
+			)); !errors.As(err, &exitError) ||
+				exitError.ExitCode() != hst.ExitFailure {
+				t.Fatalf("Cure: error = %v, want init exit status 1", err)
+			}
+
+			// the testtool is not deterministic
+			if pathname, checksum, err := c.Cure(testtool); err != nil {
+				t.Fatalf("Cure: error = %v", err)
+			} else if err = os.Remove(pathname.String()); err != nil {
+				t.Fatal(err)
+			} else {
+				p := base.Append(
+					"checksum",
+					pkg.Encode(checksum),
+				)
+				if err = os.Chmod(p.Append("bin").String(), 0700); err != nil {
+					t.Fatal(err)
+				}
+				if err = os.Chmod(p.String(), 0700); err != nil {
+					t.Fatal(err)
+				}
+				if err = os.RemoveAll(p.String()); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}, pkg.MustDecode("7PoPpWLjFPXIymbuIYLZAzOpCYr-2PN4CZ11jFdO-mDlnZNgFO3JyOtK8HW8Jxvm")},
+	})
+}

internal/pkg/pkg.go

@@ -183,6 +183,8 @@ const (
 	KindHTTPGet Kind = iota
 	// KindTar is the kind of [Artifact] returned by [NewTar].
 	KindTar
+	// KindExec is the kind of [Artifact] returned by [NewExec].
+	KindExec
 )
 
 // Ident returns a deterministic identifier for the supplied params and

internal/pkg/pkg_test.go

@@ -23,6 +23,8 @@ import (
 	"hakurei.app/internal/pkg"
 )
 
+func TestMain(m *testing.M) { container.TryArgv0(nil); os.Exit(m.Run()) }
+
 // overrideIdent overrides the ID method of [Artifact].
 type overrideIdent struct {
 	id pkg.ID
@@ -251,13 +253,16 @@ type cureStep struct {
 	err      error
 }
 
+// ignorePathname is passed to cureMany to skip the pathname check.
+var ignorePathname = check.MustAbs("/\x00")
+
 // cureMany cures many artifacts against a [Cache] and checks their outcomes.
 func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) {
 	for _, step := range steps {
 		t.Log("cure step:", step.name)
 		if pathname, checksum, err := c.Cure(step.a); !reflect.DeepEqual(err, step.err) {
 			t.Fatalf("Cure: error = %v, want %v", err, step.err)
-		} else if !pathname.Is(step.pathname) {
+		} else if step.pathname != ignorePathname && !pathname.Is(step.pathname) {
 			t.Fatalf("Cure: pathname = %q, want %q", pathname, step.pathname)
 		} else if checksum != step.checksum {
 			t.Fatalf("Cure: checksum = %s, want %s", pkg.Encode(checksum), pkg.Encode(step.checksum))

internal/pkg/testdata/main.go

@@ -0,0 +1,103 @@
+//go:build testtool
+
+package main
+
+import (
+	"log"
+	"os"
+	"path"
+	"slices"
+
+	"hakurei.app/container/fhs"
+	"hakurei.app/container/vfs"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("testtool: ")
+
+	if wantArgs := []string{"testtool"}; !slices.Equal(os.Args, wantArgs) {
+		log.Fatalf("Args: %q, want %q", os.Args, wantArgs)
+	}
+	if wantEnv := []string{
+		"HAKUREI_TEST=1",
+	}; !slices.Equal(wantEnv, os.Environ()) {
+		log.Fatalf("Environ: %q, want %q", os.Environ(), wantEnv)
+	}
+	const wantExec = "/opt/bin/testtool"
+	if got, err := os.Executable(); err != nil {
+		log.Fatalf("Executable: error = %v", err)
+	} else if got != wantExec {
+		log.Fatalf("Executable: %q, want %q", got, wantExec)
+	}
+
+	var m *vfs.MountInfo
+	if f, err := os.Open(fhs.Proc + "self/mountinfo"); err != nil {
+		log.Fatalf("Open: error = %v", err)
+	} else {
+		err = vfs.NewMountInfoDecoder(f).Decode(&m)
+		closeErr := f.Close()
+		if err != nil {
+			log.Fatalf("Decode: error = %v", err)
+		}
+		if closeErr != nil {
+			log.Fatalf("Close: error = %v", err)
+		}
+	}
+
+	log.Println(m)
+	if m.Root != "/sysroot" || m.Target != "/" {
+		log.Fatal("unexpected root mount entry")
+	}
+	next := func() { m = m.Next; log.Println(m) }
+
+	next()
+	if path.Base(m.Root) != "OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb" {
+		log.Fatal("unexpected file artifact checksum")
+	}
+
+	next()
+	if path.Base(m.Root) != "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU" {
+		log.Fatal("unexpected artifact checksum")
+	}
+
+	next() // testtool artifact
+
+	next()
+	if path.Base(m.Root) !=
+		"oHuqV7p0v1Vd8IdAzjyYM8sfCS0P2LR5tfv5cb6Gbf2ZWUm8Ec-7hYPJ_qr183m7" || m.Target != "/work" {
+		log.Fatal("unexpected work mount entry")
+	}
+
+	next()
+	if path.Base(m.Root) !=
+		"oHuqV7p0v1Vd8IdAzjyYM8sfCS0P2LR5tfv5cb6Gbf2ZWUm8Ec-7hYPJ_qr183m7" || m.Target != "/tmp" {
+		log.Fatal("unexpected temp mount entry")
+	}
+
+	next()
+	if m.Root != "/" || m.Target != "/proc" || m.Source != "proc" || m.FsType != "proc" {
+		log.Fatal("unexpected proc mount entry")
+	}
+
+	next()
+	if m.Root != "/" || m.Target != "/dev" || m.Source != "devtmpfs" || m.FsType != "tmpfs" {
+		log.Fatal("unexpected dev mount entry")
+	}
+
+	for i := 0; i < 9; i++ { // private /dev entries
+		next()
+	}
+
+	if m.Next != nil {
+		log.Println("unexpected extra mount entries")
+		for m.Next != nil {
+			next()
+		}
+		os.Exit(1)
+	}
+
+	if err := os.WriteFile("check", []byte{0}, 0400); err != nil {
+		log.Fatal(err)
+	}
+}