internal/rosa: remove legacy exec helper

This change also adds a fourth stage to the bootstrap machinery, offering more correct toolchain validation that works around LLVM dynamic linking flaws.

Signed-off-by: Ophestra <cat@gensokyo.uk>

cmd/mbf/main.go

@@ -616,7 +616,7 @@ func main() {
 
 	{
 		c.NewCommand(
-			"stage3",
+			"stage4",
 			"Check for toolchain 3-stage non-determinism",
 			func(args []string) (err error) {
 				var (
@@ -636,21 +636,31 @@ func main() {
 
 				if err = cm.Do(func(cache *pkg.Cache) (err error) {
 					_, llvm := rosa.Native().New(rosa.Std - 1).Load(_llvm)
-					pathname, checksum[0], err = cache.Cure(llvm)
+					pathname, _, err = cache.Cure(llvm)
 					return
 				}); err != nil {
 					return
 				}
 				log.Println("stage2:", pathname)
+
 				if err = cm.Do(func(cache *pkg.Cache) (err error) {
 					_, llvm := rosa.Native().New(rosa.Std).Load(_llvm)
-					pathname, checksum[1], err = cache.Cure(llvm)
+					pathname, checksum[0], err = cache.Cure(llvm)
 					return
 				}); err != nil {
 					return
 				}
 				log.Println("stage3:", pathname)
 
+				if err = cm.Do(func(cache *pkg.Cache) (err error) {
+					_, llvm := rosa.Native().New(rosa.Stage3).Load(_llvm)
+					pathname, checksum[1], err = cache.Cure(llvm)
+					return
+				}); err != nil {
+					return
+				}
+				log.Println("stage4:", pathname)
+
 				if checksum[0] != checksum[1] {
 					err = &pkg.ChecksumMismatchError{
 						Got:  checksum[0].Value(),
@@ -658,7 +668,7 @@ func main() {
 					}
 				} else {
 					log.Println(
-						"stage2 is identical to stage3",
+						"stage3 is identical to stage4",
 						"("+pkg.Encode(checksum[0].Value())+")",
 					)
 				}

internal/rosa/cmake.go

@@ -35,7 +35,7 @@ type CMakeHelper struct {
 var _ Helper = new(CMakeHelper)
 
 // extra returns the cmake handle alongside either ninja or make.
-func (attr *CMakeHelper) extra(int) P {
+func (attr *CMakeHelper) extra() P {
 	if attr != nil && attr.Make {
 		return P{_cmake, _make}
 	}

internal/rosa/git.go

@@ -17,14 +17,14 @@ func (t Toolchain) NewViaGit(
 	url, rev string,
 	checksum pkg.Checksum,
 ) pkg.Artifact {
-	return t.NewPackage(strings.TrimSuffix(
+	return t.New(strings.TrimSuffix(
 		path.Base(url),
 		".git",
 	)+"-src", path.Base(rev), nil, &PackageAttr{
 		KnownChecksum: &checksum,
 
-		Flag:  THostNet,
-		Paths: []pkg.ExecPath{resolvconf()},
+		HostNet: true,
+		Paths:   []pkg.ExecPath{resolvconf()},
 	}, &GenericHelper{
 		Build: `
 git \

internal/rosa/go.go

@@ -13,7 +13,7 @@ func (t Toolchain) newGo(
 	script string,
 	boot ...pkg.Artifact,
 ) pkg.Artifact {
-	return t.NewPackage("go", version, newTar(
+	return t.New("go", version, newTar(
 		"https://go.dev/dl/go"+version+".src.tar.gz",
 		checksum,
 		pkg.Gzip,
@@ -75,7 +75,7 @@ func init() {
 		)
 		switch t.arch {
 		case "amd64":
-			bootstrapEarly = []pkg.Artifact{t.NewPackage("go", "1.4-bootstrap", newTar(
+			bootstrapEarly = []pkg.Artifact{t.New("go", "1.4-bootstrap", newTar(
 				"https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz",
 				"8o9JL_ToiQKadCTb04nvBDkp8O1xiWOolAxVEqaTGodieNe4lOFEjlOxN3bwwe23",
 				pkg.Gzip,

internal/rosa/llvm.go

@@ -184,8 +184,8 @@ func init() {
 			}...)
 		}
 
-		return &meta, t.NewPackage("llvm", meta.Version, source, &PackageAttr{
-			Flag: TExclusive,
+		return &meta, t.New("llvm", meta.Version, source, &PackageAttr{
+			Exclusive: true,
 			Bin: []string{
 				"chmod",
 				"mkdir",

internal/rosa/make.go

@@ -52,15 +52,16 @@ type MakeHelper struct {
 var _ Helper = new(MakeHelper)
 
 // extra returns make and other optional dependencies.
-func (attr *MakeHelper) extra(flag int) P {
-	extra := P{_make}
-	if (attr == nil || !attr.OmitDefaults) && flag&TEarly == 0 {
-		extra = append(extra,
+func (attr *MakeHelper) extra() P {
+	if attr == nil || !attr.OmitDefaults {
+		return P{
+			_make,
+
 			_awk,
 			_coreutils,
-		)
+		}
 	}
-	return extra
+	return P{_make}
 }
 
 // wantsChmod returns whether the build system needs to be generated.

internal/rosa/meson.go

@@ -35,7 +35,7 @@ type MesonHelper struct {
 var _ Helper = new(MesonHelper)
 
 // extra returns hardcoded meson runtime dependencies.
-func (attr *MesonHelper) extra(int) P {
+func (attr *MesonHelper) extra() P {
 	if !attr.isPython() {
 		return P{_muon}
 	}

internal/rosa/package/bison.az

@@ -10,7 +10,9 @@ package bison {
 		compress = gzip;
 	};
 
+	toyboxEarly = true;
 	exec = make {
+		omitDefaults = true;
 		check = [
 			"TESTSUITEFLAGS=" + jobsFlagE + "' " + skipGNUTests {
 				tests = [
@@ -24,6 +26,5 @@ package bison {
 		];
 	};
 
-	toyboxEarly = true;
 	inputs = [ m4 ];
 }

internal/rosa/package/byacc.az

@@ -18,5 +18,7 @@ package byacc {
 	writable = true;
 	chmod = true;
 
-	exec = make {};
+	exec = make {
+		omitDefaults = true;
+	};
 }

internal/rosa/package/gnu.az

@@ -17,9 +17,11 @@ package m4 {
 chmod +w tests/test-c32ispunct.sh && echo '#!/bin/sh' > tests/test-c32ispunct.sh
 `;
 
-	exec = make {};
-
 	toyboxEarly = true;
+	exec = make {
+		omitDefaults = true;
+	};
+
 	inputs = [
 		diffutils,
 
@@ -194,7 +196,9 @@ test_disable '#!/bin/sh' tests/cmp
 `;
 
 	toyboxEarly = true;
-	exec = make {};
+	exec = make {
+		omitDefaults = true;
+	};
 }
 
 package patch {
@@ -218,7 +222,9 @@ test_disable '#!/bin/sh' tests/need-filename
 `;
 
 	toyboxEarly = true;
-	exec = make {};
+	exec = make {
+		omitDefaults = true;
+	};
 }
 
 package grep {
@@ -328,6 +334,7 @@ package bash {
 
 	toyboxEarly = true;
 	exec = make {
+		omitDefaults = true;
 		configure = {
 			"without-bash-malloc";
 		};
@@ -364,6 +371,7 @@ test_disable 'int main(){return 0;}' gnulib-tests/test-lchown.c
 `;
 
 	exec = make {
+		omitDefaults = true;
 		configure = {
 			"enable-single-binary": "symlinks";
 		};

internal/rosa/package/kernel/package.az

@@ -25,6 +25,7 @@ package kernel-headers {
 	toyboxEarly = true;
 
 	exec = make {
+		omitDefaults = true;
 		skipConfigure = true;
 		make = [
 			"-f /usr/src/kernel-headers/Makefile",

internal/rosa/package/rsync.az

@@ -15,6 +15,7 @@ package rsync {
 	toyboxEarly = true;
 
 	exec = make {
+		omitDefaults = true;
 		configure = {
 			"disable-openssl";
 			"disable-xxhash";

internal/rosa/perl.go

@@ -9,7 +9,7 @@ type MakeMakerHelper struct {
 }
 
 // extra returns perl.
-func (*MakeMakerHelper) extra(int) P { return P{_perl, _make} }
+func (*MakeMakerHelper) extra() P { return P{_perl, _make} }
 
 // wantsChmod returns true.
 func (*MakeMakerHelper) wantsChmod() bool { return true }

internal/rosa/python.go

@@ -26,7 +26,7 @@ type PipHelper struct {
 var _ Helper = new(PipHelper)
 
 // extra returns python, or pytest if defaults are assumed.
-func (attr *PipHelper) extra(int) P {
+func (attr *PipHelper) extra() P {
 	if attr == nil || (!attr.SkipCheck && attr.Check == "") {
 		return P{_pythonPyTest}
 	}

internal/rosa/rosa.go

@@ -121,6 +121,12 @@ const (
 	// Std denotes the standard Rosa OS toolchain.
 	Std
 
+	// Stage3 denotes the stage3 Rosa OS toolchain built on [Std]. Software
+	// built on this toolchain should be identical to [Std]. This is generally
+	// for validation of LLVM 3-stage nondeterminism only, due to dynamic
+	// linking complications.
+	Stage3
+
 	// _stageEnd is the total number of stages available and does not denote a
 	// valid toolchain.
 	_stageEnd
@@ -149,7 +155,7 @@ func (t Stage) isIntermediate() bool {
 // isStd returns whether t is considered functionally equivalent to [Std].
 func (t Stage) isStd() bool {
 	switch t {
-	case stageStdGentoo, Std:
+	case stageStdGentoo, Std, Stage3:
 		return true
 	default:
 		return false
@@ -198,17 +204,6 @@ const scriptName = "all"
 // build script under.
 var absCureScript = AbsSystem.Append(scriptName)
 
-const (
-	// TExclusive denotes an exclusive [pkg.Artifact].
-	TExclusive = 1 << iota
-	// TEarly hints for an early variant of toybox to be used when available.
-	TEarly
-	// TNoToolchain excludes the LLVM toolchain.
-	TNoToolchain
-	// THostNet arranges for a [pkg.KindExecNet] to be created.
-	THostNet
-)
-
 var (
 	_stage0Dist  = H("stage0-dist")
 	_mksh        = H("mksh")
@@ -218,102 +213,6 @@ var (
 	_patch       = H("patch")
 )
 
-// New returns a [pkg.Artifact] based on a [Toolchain] via s.
-func (t Toolchain) New(
-	name string,
-	flag int,
-	extra []pkg.Artifact,
-	knownChecksum *pkg.Checksum,
-	env []string,
-	script string,
-
-	paths ...pkg.ExecPath,
-) pkg.Artifact {
-	const lcMessages = "LC_MESSAGES=C.UTF-8"
-
-	var support []pkg.Artifact
-	switch t.stage {
-	case stageGentoo, stageEarly:
-		name += "-boot"
-		support = append(support, extra...)
-		support = append(support, NewEtc(true))
-		if t.stage == stageEarly {
-			_, a := t.MustLoad(_stage0Dist)
-			support = append(support, a)
-		} else if t.gentooStage3 == nil {
-			panic(os.ErrInvalid)
-		} else {
-			support = append(support,
-				t.gentooStage3,
-				gentooOverlay{t.gentooStage3},
-			)
-			env = append(env,
-				"CC=clang",
-				"CXX=clang++",
-			)
-		}
-		env = fixupEnviron(env, []string{
-			EnvTriple + "=" + t.triple(),
-			lcMessages,
-			"LDFLAGS=" + t.earlyLDFLAGS(true),
-		}, "/system/bin",
-			"/usr/bin",
-		)
-
-	case stageIntermediateGentoo, stageStdGentoo,
-		stageIntermediate, Std:
-		if t.stage.isIntermediate() {
-			name += "-std"
-		}
-
-		toybox := _toybox
-		if flag&TEarly != 0 {
-			toybox = _toyboxEarly
-		}
-
-		base := _llvm
-		if flag&TNoToolchain != 0 {
-			base = _musl
-		}
-
-		support = slices.Concat(extra, t.S.New(t.stage-1).Append([]pkg.Artifact{
-			NewEtc(false),
-		},
-			base,
-			_mksh,
-			toybox,
-		))
-		env = fixupEnviron(env, []string{
-			EnvTriple + "=" + t.triple(),
-			lcMessages,
-		}, "/system/bin", "/bin")
-
-	default:
-		panic("unsupported toolchain " + strconv.Itoa(int(t.stage)))
-	}
-
-	return pkg.NewExec(
-		name, t.arch, knownChecksum, pkg.ExecTimeoutMax,
-		flag&THostNet != 0,
-		flag&TExclusive != 0,
-		fhs.AbsRoot, env,
-		absCureScript,
-		nil,
-
-		slices.Concat([]pkg.ExecPath{pkg.Path(
-			fhs.AbsRoot, true,
-			support...,
-		), pkg.Path(
-			absCureScript, false,
-			pkg.NewFile(scriptName, []byte(
-				"#!/system/bin/sh\n"+
-					"set -eu -o pipefail\n"+
-					script,
-			)),
-		)}, paths)...,
-	)
-}
-
 // HasStageEarly returns whether a stage0 distribution is available.
 func (s *S) HasStageEarly() (ok bool) {
 	func() {
@@ -364,7 +263,7 @@ function apply {
 		}
 	}
 
-	return t.NewPackage(name+"-src", Unversioned, nil, &PackageAttr{
+	return t.New(name+"-src", Unversioned, nil, &PackageAttr{
 		Paths: paths,
 	}, &GenericHelper{
 		Build: buf.String(),
@@ -379,7 +278,7 @@ const helperInPlace = "\x00"
 // Helper is a build system helper for [Toolchain.NewPackage].
 type Helper interface {
 	// extra returns helper-specific dependencies.
-	extra(flag int) P
+	extra() P
 
 	// wantsChmod returns whether the source directory should be made writable.
 	wantsChmod() bool
@@ -421,12 +320,19 @@ type PackageAttr struct {
 	// Whether to replace /usr/bin/ with a symlink to /bin/.
 	PopulateUsrBin bool
 
+	// Hint for an early variant of toybox to be used when available.
+	Early bool
+	// Exclude the LLVM toolchain.
+	NoToolchain bool
+	// Whether the resulting [pkg.Artifact] is exclusive.
+	Exclusive bool
+	// Whether to create [pkg.KindExecNet] instead of [pkg.KindExec].
+	HostNet bool
+
 	// Unregistered extras.
 	Extra []pkg.Artifact
 	// Passed through to [Toolchain.New], before source.
 	Paths []pkg.ExecPath
-	// Passed through to [Toolchain.New].
-	Flag int
 }
 
 // pa holds whether an [ArtifactH] is present.
@@ -467,8 +373,8 @@ func (t Toolchain) Append(a []pkg.Artifact, handles ...ArtifactH) []pkg.Artifact
 	return a
 }
 
-// NewPackage constructs a [pkg.Artifact] via a build system helper.
-func (t Toolchain) NewPackage(
+// New constructs a [pkg.Artifact] via a build system helper.
+func (t Toolchain) New(
 	name, version string,
 	source pkg.Artifact,
 	attr *PackageAttr,
@@ -486,25 +392,96 @@ func (t Toolchain) NewPackage(
 	if version != Unversioned {
 		rn = name + "-" + version
 	}
+
+	root := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra))
+	root = append(root, attr.Extra...)
+
+	const lcMessages = "LC_MESSAGES=C.UTF-8"
+	srn := rn
+	env := slices.Clone(attr.Env)
+
+	var extraBoot []ArtifactH
+	switch t.stage {
+	case stageGentoo, stageEarly:
+		srn += "-boot"
+		root = append(root, NewEtc(true))
+		if t.stage == stageEarly {
+			extraBoot = append(extraBoot, _stage0Dist)
+		} else if t.gentooStage3 == nil {
+			panic(os.ErrInvalid)
+		} else {
+			env = append(env,
+				"CC=clang",
+				"CXX=clang++",
+			)
+		}
+		env = fixupEnviron(env, []string{
+			EnvTriple + "=" + t.triple(),
+			lcMessages,
+			"LDFLAGS=" + t.earlyLDFLAGS(true),
+		}, "/system/bin",
+			"/usr/bin",
+		)
+
+	case stageIntermediateGentoo, stageStdGentoo,
+		stageIntermediate, Std, Stage3:
+		if t.stage.isIntermediate() {
+			srn += "-std"
+		}
+
+		if t.stage == Stage3 {
+			srn += "-stage4"
+		}
+
+		toybox := _toybox
+		if attr.Early {
+			toybox = _toyboxEarly
+		}
+
+		base := _llvm
+		if attr.NoToolchain {
+			base = _musl
+		}
+
+		root = append(root, NewEtc(false))
+		extraBoot = append(extraBoot,
+			base,
+			_mksh,
+			toybox,
+		)
+		env = fixupEnviron(env, []string{
+			EnvTriple + "=" + t.triple(),
+			lcMessages,
+		}, "/system/bin", "/bin")
+
+	default:
+		panic("unsupported toolchain " + strconv.Itoa(int(t.stage)))
+	}
+
 	wantsChmod, wantsWrite := helper.wantsChmod(), helper.wantsWrite()
-	extraRes := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra))
-	extraRes = append(extraRes, attr.Extra...)
 	{
 		pv := paGet()
-		for _, p := range helper.extra(attr.Flag) {
-			extraRes = t.appendHandle(extraRes, pv, p)
+		for _, p := range helper.extra() {
+			root = t.appendHandle(root, pv, p)
 		}
 		for _, p := range extra {
-			extraRes = t.appendHandle(extraRes, pv, p)
+			root = t.appendHandle(root, pv, p)
+		}
+		boot := t.S.New(t.stage - 1)
+		for _, p := range extraBoot {
+			root = boot.appendHandle(root, pv, p)
 		}
 		paPut(pv)
 	}
 
-	var (
-		scriptEarly  string
-		sourceSuffix string
-	)
+	if t.stage == stageGentoo {
+		root = append(root,
+			t.gentooStage3,
+			gentooOverlay{t.gentooStage3},
+		)
+	}
 
+	var scriptEarly string
 	dir, create := helper.wantsDir()
 	helperScriptEarly := helper.scriptEarly()
 	if attr.EnterSource ||
@@ -529,7 +506,7 @@ cd '/usr/src/` + name + `/'
 	if source != nil {
 		paths = slices.Concat(attr.Paths, []pkg.ExecPath{
 			pkg.Path(AbsUsrSrc.Append(
-				name+sourceSuffix,
+				name,
 			), attr.Writable || wantsWrite, t.NewPatchedSource(
 				rn, source, !attr.Chmod && !wantsChmod, attr.Patches...,
 			)),
@@ -556,14 +533,25 @@ ln -s ../bin /usr/`
 		scriptEarly += " /bin/)\n"
 	}
 
-	return t.New(
-		rn,
-		attr.Flag,
-		extraRes,
-		attr.KnownChecksum,
-		attr.Env,
-		scriptEarly+helper.script(t, name),
-		paths...,
+	return pkg.NewExec(
+		srn, t.arch, attr.KnownChecksum, pkg.ExecTimeoutMax,
+		attr.HostNet,
+		attr.Exclusive,
+		fhs.AbsRoot, env,
+		absCureScript,
+		nil,
+
+		slices.Concat([]pkg.ExecPath{pkg.Path(
+			fhs.AbsRoot, true,
+			root...,
+		), pkg.Path(
+			absCureScript, false,
+			pkg.NewFile(scriptName, []byte(
+				"#!/system/bin/sh\n"+
+					"set -eu -o pipefail\n"+
+					scriptEarly+helper.script(t, name),
+			)),
+		)}, paths)...,
 	)
 }
 
@@ -581,7 +569,7 @@ type GenericHelper struct {
 var _ Helper = new(GenericHelper)
 
 // extra is a noop.
-func (*GenericHelper) extra(int) P { return nil }
+func (*GenericHelper) extra() P { return nil }
 
 // wantsChmod returns false.
 func (*GenericHelper) wantsChmod() bool { return false }

internal/rosa/state.go

@@ -989,9 +989,6 @@ func (ctx *evalContext) pf(
 		attr    PackageAttr
 		patches []string
 		files   []KV
-		excl    bool
-		early   bool
-		minimal bool
 
 		anitya  int64
 		kc      any
@@ -1020,9 +1017,9 @@ func (ctx *evalContext) pf(
 		k("populateUsrBin"): &attr.PopulateUsrBin,
 		k("patches"):        &patches,
 		k("files"):          &files,
-		k("exclusive"):      &excl,
-		k("toyboxEarly"):    &early,
-		k("minimal"):        &minimal,
+		k("exclusive"):      &attr.Exclusive,
+		k("toyboxEarly"):    &attr.Early,
+		k("minimal"):        &attr.NoToolchain,
 
 		k("checksum"): &kc,
 		k("output"):   &output,
@@ -1116,16 +1113,6 @@ func (ctx *evalContext) pf(
 		)
 	}
 
-	if excl {
-		attr.Flag |= TExclusive
-	}
-	if early {
-		attr.Flag |= TEarly
-	}
-	if minimal {
-		attr.Flag |= TNoToolchain
-	}
-
 	if kc != nil {
 		checksum, ok := kc.(string)
 		if !ok {
@@ -1162,7 +1149,7 @@ func (ctx *evalContext) pf(
 		}
 	}
 
-	v = cachedArtifact{&meta, ctx.t.NewPackage(
+	v = cachedArtifact{&meta, ctx.t.New(
 		meta.Name,
 		meta.Version,
 		source,
@@ -1175,8 +1162,8 @@ func (ctx *evalContext) pf(
 }
 
 var (
-	// ErrToplevel is returned by [S.Evaluate] when encountering a toplevel
-	// expression other than a package declaration.
+	// ErrToplevel is returned by [S.RegisterAzalea] when encountering a
+	// toplevel expression other than a package declaration.
 	ErrToplevel = errors.New("top level must only contain package declarations")
 )