diff --git a/cmd/mbf/main.go b/cmd/mbf/main.go index 6df5d741..d464fd54 100644 --- a/cmd/mbf/main.go +++ b/cmd/mbf/main.go @@ -616,7 +616,7 @@ func main() { { c.NewCommand( - "stage3", + "stage4", "Check for toolchain 3-stage non-determinism", func(args []string) (err error) { var ( @@ -636,21 +636,31 @@ func main() { if err = cm.Do(func(cache *pkg.Cache) (err error) { _, llvm := rosa.Native().New(rosa.Std - 1).Load(_llvm) - pathname, checksum[0], err = cache.Cure(llvm) + pathname, _, err = cache.Cure(llvm) return }); err != nil { return } log.Println("stage2:", pathname) + if err = cm.Do(func(cache *pkg.Cache) (err error) { _, llvm := rosa.Native().New(rosa.Std).Load(_llvm) - pathname, checksum[1], err = cache.Cure(llvm) + pathname, checksum[0], err = cache.Cure(llvm) return }); err != nil { return } log.Println("stage3:", pathname) + if err = cm.Do(func(cache *pkg.Cache) (err error) { + _, llvm := rosa.Native().New(rosa.Stage3).Load(_llvm) + pathname, checksum[1], err = cache.Cure(llvm) + return + }); err != nil { + return + } + log.Println("stage4:", pathname) + if checksum[0] != checksum[1] { err = &pkg.ChecksumMismatchError{ Got: checksum[0].Value(), @@ -658,7 +668,7 @@ func main() { } } else { log.Println( - "stage2 is identical to stage3", + "stage3 is identical to stage4", "("+pkg.Encode(checksum[0].Value())+")", ) } diff --git a/internal/rosa/cmake.go b/internal/rosa/cmake.go index 43defd87..8a67f4bc 100644 --- a/internal/rosa/cmake.go +++ b/internal/rosa/cmake.go @@ -35,7 +35,7 @@ type CMakeHelper struct { var _ Helper = new(CMakeHelper) // extra returns the cmake handle alongside either ninja or make. -func (attr *CMakeHelper) extra(int) P { +func (attr *CMakeHelper) extra() P { if attr != nil && attr.Make { return P{_cmake, _make} } diff --git a/internal/rosa/git.go b/internal/rosa/git.go index a1bd8e0d..bda380ce 100644 --- a/internal/rosa/git.go +++ b/internal/rosa/git.go @@ -17,14 +17,14 @@ func (t Toolchain) NewViaGit( url, rev string, checksum pkg.Checksum, ) pkg.Artifact { - return t.NewPackage(strings.TrimSuffix( + return t.New(strings.TrimSuffix( path.Base(url), ".git", )+"-src", path.Base(rev), nil, &PackageAttr{ KnownChecksum: &checksum, - Flag: THostNet, - Paths: []pkg.ExecPath{resolvconf()}, + HostNet: true, + Paths: []pkg.ExecPath{resolvconf()}, }, &GenericHelper{ Build: ` git \ diff --git a/internal/rosa/go.go b/internal/rosa/go.go index ebdaa8e2..bb3f4773 100644 --- a/internal/rosa/go.go +++ b/internal/rosa/go.go @@ -13,7 +13,7 @@ func (t Toolchain) newGo( script string, boot ...pkg.Artifact, ) pkg.Artifact { - return t.NewPackage("go", version, newTar( + return t.New("go", version, newTar( "https://go.dev/dl/go"+version+".src.tar.gz", checksum, pkg.Gzip, @@ -75,7 +75,7 @@ func init() { ) switch t.arch { case "amd64": - bootstrapEarly = []pkg.Artifact{t.NewPackage("go", "1.4-bootstrap", newTar( + bootstrapEarly = []pkg.Artifact{t.New("go", "1.4-bootstrap", newTar( "https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz", "8o9JL_ToiQKadCTb04nvBDkp8O1xiWOolAxVEqaTGodieNe4lOFEjlOxN3bwwe23", pkg.Gzip, diff --git a/internal/rosa/llvm.go b/internal/rosa/llvm.go index 8189b642..8a844268 100644 --- a/internal/rosa/llvm.go +++ b/internal/rosa/llvm.go @@ -184,8 +184,8 @@ func init() { }...) } - return &meta, t.NewPackage("llvm", meta.Version, source, &PackageAttr{ - Flag: TExclusive, + return &meta, t.New("llvm", meta.Version, source, &PackageAttr{ + Exclusive: true, Bin: []string{ "chmod", "mkdir", diff --git a/internal/rosa/make.go b/internal/rosa/make.go index bc237f17..8b1103d6 100644 --- a/internal/rosa/make.go +++ b/internal/rosa/make.go @@ -52,15 +52,16 @@ type MakeHelper struct { var _ Helper = new(MakeHelper) // extra returns make and other optional dependencies. -func (attr *MakeHelper) extra(flag int) P { - extra := P{_make} - if (attr == nil || !attr.OmitDefaults) && flag&TEarly == 0 { - extra = append(extra, +func (attr *MakeHelper) extra() P { + if attr == nil || !attr.OmitDefaults { + return P{ + _make, + _awk, _coreutils, - ) + } } - return extra + return P{_make} } // wantsChmod returns whether the build system needs to be generated. diff --git a/internal/rosa/meson.go b/internal/rosa/meson.go index b304ffc8..65e9fde9 100644 --- a/internal/rosa/meson.go +++ b/internal/rosa/meson.go @@ -35,7 +35,7 @@ type MesonHelper struct { var _ Helper = new(MesonHelper) // extra returns hardcoded meson runtime dependencies. -func (attr *MesonHelper) extra(int) P { +func (attr *MesonHelper) extra() P { if !attr.isPython() { return P{_muon} } diff --git a/internal/rosa/package/bison.az b/internal/rosa/package/bison.az index c732ed1b..5d231777 100644 --- a/internal/rosa/package/bison.az +++ b/internal/rosa/package/bison.az @@ -10,7 +10,9 @@ package bison { compress = gzip; }; + toyboxEarly = true; exec = make { + omitDefaults = true; check = [ "TESTSUITEFLAGS=" + jobsFlagE + "' " + skipGNUTests { tests = [ @@ -24,6 +26,5 @@ package bison { ]; }; - toyboxEarly = true; inputs = [ m4 ]; } diff --git a/internal/rosa/package/byacc.az b/internal/rosa/package/byacc.az index e79bb1ba..67dc2ad5 100644 --- a/internal/rosa/package/byacc.az +++ b/internal/rosa/package/byacc.az @@ -18,5 +18,7 @@ package byacc { writable = true; chmod = true; - exec = make {}; + exec = make { + omitDefaults = true; + }; } diff --git a/internal/rosa/package/gnu.az b/internal/rosa/package/gnu.az index 7cfcc726..d4b9b5b2 100644 --- a/internal/rosa/package/gnu.az +++ b/internal/rosa/package/gnu.az @@ -17,9 +17,11 @@ package m4 { chmod +w tests/test-c32ispunct.sh && echo '#!/bin/sh' > tests/test-c32ispunct.sh `; - exec = make {}; - toyboxEarly = true; + exec = make { + omitDefaults = true; + }; + inputs = [ diffutils, @@ -194,7 +196,9 @@ test_disable '#!/bin/sh' tests/cmp `; toyboxEarly = true; - exec = make {}; + exec = make { + omitDefaults = true; + }; } package patch { @@ -218,7 +222,9 @@ test_disable '#!/bin/sh' tests/need-filename `; toyboxEarly = true; - exec = make {}; + exec = make { + omitDefaults = true; + }; } package grep { @@ -328,6 +334,7 @@ package bash { toyboxEarly = true; exec = make { + omitDefaults = true; configure = { "without-bash-malloc"; }; @@ -364,6 +371,7 @@ test_disable 'int main(){return 0;}' gnulib-tests/test-lchown.c `; exec = make { + omitDefaults = true; configure = { "enable-single-binary": "symlinks"; }; diff --git a/internal/rosa/package/kernel/package.az b/internal/rosa/package/kernel/package.az index efe2b304..1bf17ae6 100644 --- a/internal/rosa/package/kernel/package.az +++ b/internal/rosa/package/kernel/package.az @@ -25,6 +25,7 @@ package kernel-headers { toyboxEarly = true; exec = make { + omitDefaults = true; skipConfigure = true; make = [ "-f /usr/src/kernel-headers/Makefile", diff --git a/internal/rosa/package/rsync.az b/internal/rosa/package/rsync.az index 62bb045a..8bb06bcb 100644 --- a/internal/rosa/package/rsync.az +++ b/internal/rosa/package/rsync.az @@ -15,6 +15,7 @@ package rsync { toyboxEarly = true; exec = make { + omitDefaults = true; configure = { "disable-openssl"; "disable-xxhash"; diff --git a/internal/rosa/perl.go b/internal/rosa/perl.go index 313f8dc8..a828609b 100644 --- a/internal/rosa/perl.go +++ b/internal/rosa/perl.go @@ -9,7 +9,7 @@ type MakeMakerHelper struct { } // extra returns perl. -func (*MakeMakerHelper) extra(int) P { return P{_perl, _make} } +func (*MakeMakerHelper) extra() P { return P{_perl, _make} } // wantsChmod returns true. func (*MakeMakerHelper) wantsChmod() bool { return true } diff --git a/internal/rosa/python.go b/internal/rosa/python.go index 56d23339..96f7714c 100644 --- a/internal/rosa/python.go +++ b/internal/rosa/python.go @@ -26,7 +26,7 @@ type PipHelper struct { var _ Helper = new(PipHelper) // extra returns python, or pytest if defaults are assumed. -func (attr *PipHelper) extra(int) P { +func (attr *PipHelper) extra() P { if attr == nil || (!attr.SkipCheck && attr.Check == "") { return P{_pythonPyTest} } diff --git a/internal/rosa/rosa.go b/internal/rosa/rosa.go index 77dc1592..8287795d 100644 --- a/internal/rosa/rosa.go +++ b/internal/rosa/rosa.go @@ -121,6 +121,12 @@ const ( // Std denotes the standard Rosa OS toolchain. Std + // Stage3 denotes the stage3 Rosa OS toolchain built on [Std]. Software + // built on this toolchain should be identical to [Std]. This is generally + // for validation of LLVM 3-stage nondeterminism only, due to dynamic + // linking complications. + Stage3 + // _stageEnd is the total number of stages available and does not denote a // valid toolchain. _stageEnd @@ -149,7 +155,7 @@ func (t Stage) isIntermediate() bool { // isStd returns whether t is considered functionally equivalent to [Std]. func (t Stage) isStd() bool { switch t { - case stageStdGentoo, Std: + case stageStdGentoo, Std, Stage3: return true default: return false @@ -198,17 +204,6 @@ const scriptName = "all" // build script under. var absCureScript = AbsSystem.Append(scriptName) -const ( - // TExclusive denotes an exclusive [pkg.Artifact]. - TExclusive = 1 << iota - // TEarly hints for an early variant of toybox to be used when available. - TEarly - // TNoToolchain excludes the LLVM toolchain. - TNoToolchain - // THostNet arranges for a [pkg.KindExecNet] to be created. - THostNet -) - var ( _stage0Dist = H("stage0-dist") _mksh = H("mksh") @@ -218,102 +213,6 @@ var ( _patch = H("patch") ) -// New returns a [pkg.Artifact] based on a [Toolchain] via s. -func (t Toolchain) New( - name string, - flag int, - extra []pkg.Artifact, - knownChecksum *pkg.Checksum, - env []string, - script string, - - paths ...pkg.ExecPath, -) pkg.Artifact { - const lcMessages = "LC_MESSAGES=C.UTF-8" - - var support []pkg.Artifact - switch t.stage { - case stageGentoo, stageEarly: - name += "-boot" - support = append(support, extra...) - support = append(support, NewEtc(true)) - if t.stage == stageEarly { - _, a := t.MustLoad(_stage0Dist) - support = append(support, a) - } else if t.gentooStage3 == nil { - panic(os.ErrInvalid) - } else { - support = append(support, - t.gentooStage3, - gentooOverlay{t.gentooStage3}, - ) - env = append(env, - "CC=clang", - "CXX=clang++", - ) - } - env = fixupEnviron(env, []string{ - EnvTriple + "=" + t.triple(), - lcMessages, - "LDFLAGS=" + t.earlyLDFLAGS(true), - }, "/system/bin", - "/usr/bin", - ) - - case stageIntermediateGentoo, stageStdGentoo, - stageIntermediate, Std: - if t.stage.isIntermediate() { - name += "-std" - } - - toybox := _toybox - if flag&TEarly != 0 { - toybox = _toyboxEarly - } - - base := _llvm - if flag&TNoToolchain != 0 { - base = _musl - } - - support = slices.Concat(extra, t.S.New(t.stage-1).Append([]pkg.Artifact{ - NewEtc(false), - }, - base, - _mksh, - toybox, - )) - env = fixupEnviron(env, []string{ - EnvTriple + "=" + t.triple(), - lcMessages, - }, "/system/bin", "/bin") - - default: - panic("unsupported toolchain " + strconv.Itoa(int(t.stage))) - } - - return pkg.NewExec( - name, t.arch, knownChecksum, pkg.ExecTimeoutMax, - flag&THostNet != 0, - flag&TExclusive != 0, - fhs.AbsRoot, env, - absCureScript, - nil, - - slices.Concat([]pkg.ExecPath{pkg.Path( - fhs.AbsRoot, true, - support..., - ), pkg.Path( - absCureScript, false, - pkg.NewFile(scriptName, []byte( - "#!/system/bin/sh\n"+ - "set -eu -o pipefail\n"+ - script, - )), - )}, paths)..., - ) -} - // HasStageEarly returns whether a stage0 distribution is available. func (s *S) HasStageEarly() (ok bool) { func() { @@ -364,7 +263,7 @@ function apply { } } - return t.NewPackage(name+"-src", Unversioned, nil, &PackageAttr{ + return t.New(name+"-src", Unversioned, nil, &PackageAttr{ Paths: paths, }, &GenericHelper{ Build: buf.String(), @@ -379,7 +278,7 @@ const helperInPlace = "\x00" // Helper is a build system helper for [Toolchain.NewPackage]. type Helper interface { // extra returns helper-specific dependencies. - extra(flag int) P + extra() P // wantsChmod returns whether the source directory should be made writable. wantsChmod() bool @@ -421,12 +320,19 @@ type PackageAttr struct { // Whether to replace /usr/bin/ with a symlink to /bin/. PopulateUsrBin bool + // Hint for an early variant of toybox to be used when available. + Early bool + // Exclude the LLVM toolchain. + NoToolchain bool + // Whether the resulting [pkg.Artifact] is exclusive. + Exclusive bool + // Whether to create [pkg.KindExecNet] instead of [pkg.KindExec]. + HostNet bool + // Unregistered extras. Extra []pkg.Artifact // Passed through to [Toolchain.New], before source. Paths []pkg.ExecPath - // Passed through to [Toolchain.New]. - Flag int } // pa holds whether an [ArtifactH] is present. @@ -467,8 +373,8 @@ func (t Toolchain) Append(a []pkg.Artifact, handles ...ArtifactH) []pkg.Artifact return a } -// NewPackage constructs a [pkg.Artifact] via a build system helper. -func (t Toolchain) NewPackage( +// New constructs a [pkg.Artifact] via a build system helper. +func (t Toolchain) New( name, version string, source pkg.Artifact, attr *PackageAttr, @@ -486,25 +392,96 @@ func (t Toolchain) NewPackage( if version != Unversioned { rn = name + "-" + version } + + root := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra)) + root = append(root, attr.Extra...) + + const lcMessages = "LC_MESSAGES=C.UTF-8" + srn := rn + env := slices.Clone(attr.Env) + + var extraBoot []ArtifactH + switch t.stage { + case stageGentoo, stageEarly: + srn += "-boot" + root = append(root, NewEtc(true)) + if t.stage == stageEarly { + extraBoot = append(extraBoot, _stage0Dist) + } else if t.gentooStage3 == nil { + panic(os.ErrInvalid) + } else { + env = append(env, + "CC=clang", + "CXX=clang++", + ) + } + env = fixupEnviron(env, []string{ + EnvTriple + "=" + t.triple(), + lcMessages, + "LDFLAGS=" + t.earlyLDFLAGS(true), + }, "/system/bin", + "/usr/bin", + ) + + case stageIntermediateGentoo, stageStdGentoo, + stageIntermediate, Std, Stage3: + if t.stage.isIntermediate() { + srn += "-std" + } + + if t.stage == Stage3 { + srn += "-stage4" + } + + toybox := _toybox + if attr.Early { + toybox = _toyboxEarly + } + + base := _llvm + if attr.NoToolchain { + base = _musl + } + + root = append(root, NewEtc(false)) + extraBoot = append(extraBoot, + base, + _mksh, + toybox, + ) + env = fixupEnviron(env, []string{ + EnvTriple + "=" + t.triple(), + lcMessages, + }, "/system/bin", "/bin") + + default: + panic("unsupported toolchain " + strconv.Itoa(int(t.stage))) + } + wantsChmod, wantsWrite := helper.wantsChmod(), helper.wantsWrite() - extraRes := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra)) - extraRes = append(extraRes, attr.Extra...) { pv := paGet() - for _, p := range helper.extra(attr.Flag) { - extraRes = t.appendHandle(extraRes, pv, p) + for _, p := range helper.extra() { + root = t.appendHandle(root, pv, p) } for _, p := range extra { - extraRes = t.appendHandle(extraRes, pv, p) + root = t.appendHandle(root, pv, p) + } + boot := t.S.New(t.stage - 1) + for _, p := range extraBoot { + root = boot.appendHandle(root, pv, p) } paPut(pv) } - var ( - scriptEarly string - sourceSuffix string - ) + if t.stage == stageGentoo { + root = append(root, + t.gentooStage3, + gentooOverlay{t.gentooStage3}, + ) + } + var scriptEarly string dir, create := helper.wantsDir() helperScriptEarly := helper.scriptEarly() if attr.EnterSource || @@ -529,7 +506,7 @@ cd '/usr/src/` + name + `/' if source != nil { paths = slices.Concat(attr.Paths, []pkg.ExecPath{ pkg.Path(AbsUsrSrc.Append( - name+sourceSuffix, + name, ), attr.Writable || wantsWrite, t.NewPatchedSource( rn, source, !attr.Chmod && !wantsChmod, attr.Patches..., )), @@ -556,14 +533,25 @@ ln -s ../bin /usr/` scriptEarly += " /bin/)\n" } - return t.New( - rn, - attr.Flag, - extraRes, - attr.KnownChecksum, - attr.Env, - scriptEarly+helper.script(t, name), - paths..., + return pkg.NewExec( + srn, t.arch, attr.KnownChecksum, pkg.ExecTimeoutMax, + attr.HostNet, + attr.Exclusive, + fhs.AbsRoot, env, + absCureScript, + nil, + + slices.Concat([]pkg.ExecPath{pkg.Path( + fhs.AbsRoot, true, + root..., + ), pkg.Path( + absCureScript, false, + pkg.NewFile(scriptName, []byte( + "#!/system/bin/sh\n"+ + "set -eu -o pipefail\n"+ + scriptEarly+helper.script(t, name), + )), + )}, paths)..., ) } @@ -581,7 +569,7 @@ type GenericHelper struct { var _ Helper = new(GenericHelper) // extra is a noop. -func (*GenericHelper) extra(int) P { return nil } +func (*GenericHelper) extra() P { return nil } // wantsChmod returns false. func (*GenericHelper) wantsChmod() bool { return false } diff --git a/internal/rosa/state.go b/internal/rosa/state.go index 516efd81..9d864a53 100644 --- a/internal/rosa/state.go +++ b/internal/rosa/state.go @@ -989,9 +989,6 @@ func (ctx *evalContext) pf( attr PackageAttr patches []string files []KV - excl bool - early bool - minimal bool anitya int64 kc any @@ -1020,9 +1017,9 @@ func (ctx *evalContext) pf( k("populateUsrBin"): &attr.PopulateUsrBin, k("patches"): &patches, k("files"): &files, - k("exclusive"): &excl, - k("toyboxEarly"): &early, - k("minimal"): &minimal, + k("exclusive"): &attr.Exclusive, + k("toyboxEarly"): &attr.Early, + k("minimal"): &attr.NoToolchain, k("checksum"): &kc, k("output"): &output, @@ -1116,16 +1113,6 @@ func (ctx *evalContext) pf( ) } - if excl { - attr.Flag |= TExclusive - } - if early { - attr.Flag |= TEarly - } - if minimal { - attr.Flag |= TNoToolchain - } - if kc != nil { checksum, ok := kc.(string) if !ok { @@ -1162,7 +1149,7 @@ func (ctx *evalContext) pf( } } - v = cachedArtifact{&meta, ctx.t.NewPackage( + v = cachedArtifact{&meta, ctx.t.New( meta.Name, meta.Version, source, @@ -1175,8 +1162,8 @@ func (ctx *evalContext) pf( } var ( - // ErrToplevel is returned by [S.Evaluate] when encountering a toplevel - // expression other than a package declaration. + // ErrToplevel is returned by [S.RegisterAzalea] when encountering a + // toplevel expression other than a package declaration. ErrToplevel = errors.New("top level must only contain package declarations") )