4 Commits
Author SHA1 Message Date
cat 6487b41d92 internal/rosa: remove legacy exec helper
Test / Create distribution (push) Successful in 1m51s
Test / Sandbox (push) Successful in 5m27s
Test / Hakurei (push) Failing after 5m51s
Test / Sandbox (race detector) (push) Successful in 8m21s
Test / Hakurei (race detector) (push) Successful in 9m33s
Test / ShareFS (push) Successful in 10m32s
Test / Flake checks (push) Skipped
This change also adds a fourth stage to the bootstrap machinery, offering more correct toolchain validation that works around LLVM dynamic linking flaws.

Signed-off-by: Ophestra <cat@gensokyo.uk>
2026-07-30 00:18:22 +09:00
cat 7908fb9583 internal/rosa/llvm: exclude runtime symbols
Test / Hakurei (push) Successful in 4m44s
Test / Sandbox (push) Successful in 3m5s
Test / Sandbox (race detector) (push) Successful in 5m55s
Test / ShareFS (push) Successful in 7m22s
Test / Create distribution (push) Successful in 1m0s
Test / Hakurei (race detector) (push) Successful in 6m22s
Test / Flake checks (push) Successful in 1m13s
These leak by default when static linking, which is considered unacceptable for many libraries.

Signed-off-by: Ophestra <cat@gensokyo.uk>
2026-07-29 03:00:59 +09:00
cat 95aa7c205c internal/rosa: remove calls to low-level exec helper
Test / Create distribution (push) Successful in 53s
Test / Sandbox (push) Successful in 2m54s
Test / Hakurei (push) Successful in 4m32s
Test / Sandbox (race detector) (push) Successful in 5m48s
Test / Hakurei (race detector) (push) Successful in 6m59s
Test / ShareFS (push) Successful in 7m29s
Test / Flake checks (push) Successful in 1m12s
This method predates the helper infrastructure, it is awkward to use and difficult to maintain for the replacement bootstrap mechanism. This change prepares for its removal.

Signed-off-by: Ophestra <cat@gensokyo.uk>
2026-07-27 18:24:38 +09:00
cat bd0cc3656f internal/rosa/llvm: statically link libc++
Test / Hakurei (race detector) (push) Successful in 6m54s
Test / Sandbox (push) Successful in 2m53s
Test / Sandbox (race detector) (push) Successful in 5m58s
Test / ShareFS (push) Successful in 7m25s
Test / Create distribution (push) Successful in 53s
Test / Hakurei (push) Successful in 2m51s
Test / Flake checks (push) Successful in 1m9s
This avoids unnecessarily depending on the toolchain.

Signed-off-by: Ophestra <cat@gensokyo.uk>
2026-07-27 18:15:00 +09:00
17 changed files with 211 additions and 186 deletions
+14 -4
View File
@@ -616,7 +616,7 @@ func main() {
{
c.NewCommand(
"stage3",
"stage4",
"Check for toolchain 3-stage non-determinism",
func(args []string) (err error) {
var (
@@ -636,21 +636,31 @@ func main() {
if err = cm.Do(func(cache *pkg.Cache) (err error) {
_, llvm := rosa.Native().New(rosa.Std - 1).Load(_llvm)
pathname, checksum[0], err = cache.Cure(llvm)
pathname, _, err = cache.Cure(llvm)
return
}); err != nil {
return
}
log.Println("stage2:", pathname)
if err = cm.Do(func(cache *pkg.Cache) (err error) {
_, llvm := rosa.Native().New(rosa.Std).Load(_llvm)
pathname, checksum[1], err = cache.Cure(llvm)
pathname, checksum[0], err = cache.Cure(llvm)
return
}); err != nil {
return
}
log.Println("stage3:", pathname)
if err = cm.Do(func(cache *pkg.Cache) (err error) {
_, llvm := rosa.Native().New(rosa.Stage3).Load(_llvm)
pathname, checksum[1], err = cache.Cure(llvm)
return
}); err != nil {
return
}
log.Println("stage4:", pathname)
if checksum[0] != checksum[1] {
err = &pkg.ChecksumMismatchError{
Got: checksum[0].Value(),
@@ -658,7 +668,7 @@ func main() {
}
} else {
log.Println(
"stage2 is identical to stage3",
"stage3 is identical to stage4",
"("+pkg.Encode(checksum[0].Value())+")",
)
}
+1 -1
View File
@@ -35,7 +35,7 @@ type CMakeHelper struct {
var _ Helper = new(CMakeHelper)
// extra returns the cmake handle alongside either ninja or make.
func (attr *CMakeHelper) extra(int) P {
func (attr *CMakeHelper) extra() P {
if attr != nil && attr.Make {
return P{_cmake, _make}
}
+14 -7
View File
@@ -20,19 +20,26 @@ func (t Toolchain) NewViaGit(
return t.New(strings.TrimSuffix(
path.Base(url),
".git",
)+"-src-"+path.Base(rev), THostNet, t.Append(nil,
_nssCACert,
_git,
), &checksum, nil, `
)+"-src", path.Base(rev), nil, &PackageAttr{
KnownChecksum: &checksum,
HostNet: true,
Paths: []pkg.ExecPath{resolvconf()},
}, &GenericHelper{
Build: `
git \
-c advice.detachedHead=false \
clone \
--depth=1 \
--revision=`+rev+` \
--revision=` + rev + ` \
--shallow-submodules \
--recurse-submodules \
`+url+` \
` + url + ` \
/work
rm -rf /work/.git
`, resolvconf())
`,
},
_nssCACert,
_git,
)
}
+2 -2
View File
@@ -13,7 +13,7 @@ func (t Toolchain) newGo(
script string,
boot ...pkg.Artifact,
) pkg.Artifact {
return t.NewPackage("go", version, newTar(
return t.New("go", version, newTar(
"https://go.dev/dl/go"+version+".src.tar.gz",
checksum,
pkg.Gzip,
@@ -75,7 +75,7 @@ func init() {
)
switch t.arch {
case "amd64":
bootstrapEarly = []pkg.Artifact{t.NewPackage("go", "1.4-bootstrap", newTar(
bootstrapEarly = []pkg.Artifact{t.New("go", "1.4-bootstrap", newTar(
"https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz",
"8o9JL_ToiQKadCTb04nvBDkp8O1xiWOolAxVEqaTGodieNe4lOFEjlOxN3bwwe23",
pkg.Gzip,
+17 -3
View File
@@ -83,9 +83,11 @@ func init() {
{"LIBCXX_ENABLE_STATIC_ABI_LIBRARY", "OFF"},
{"LIBCXX_HAS_MUSL_LIBC", "ON"},
{"LIBCXX_HARDENING_MODE", "fast"},
{"LIBCXX_STATICALLY_LINK_ABI_IN_STATIC_LIBRARY", "ON"},
{"LIBCXXABI_USE_LLVM_UNWINDER", "ON"},
{"LIBCXXABI_ENABLE_STATIC_UNWINDER", "OFF"},
{"LIBCXXABI_ENABLE_STATIC_UNWINDER", "ON"},
{"LIBCXXABI_USE_COMPILER_RT", "ON"},
{"LIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY", "ON"},
{"LLVM_INSTALL_BINUTILS_SYMLINKS", "ON"},
{"LLVM_INSTALL_UTILS", "ON"},
{"LLVM_BUILD_LLVM_DYLIB", "ON"},
@@ -138,6 +140,10 @@ func init() {
"CodeGen/PowerPC/ppc-tmmintrin.c",
"CodeGen/PowerPC/ppc-smmintrin.c",
"CodeGenCUDA/amdgpu-alias-undef-symbols.cu",
// clang: deadlocks with LIBCXXABI_ENABLE_STATIC_UNWINDER
"global-value-regex",
"global-hex-value-regex",
"check-globals",
// cxx: fails on musl
"close.dont-get-rid-of-buffer",
"re/re.traits",
@@ -178,8 +184,8 @@ func init() {
}...)
}
return &meta, t.NewPackage("llvm", meta.Version, source, &PackageAttr{
Flag: TExclusive,
return &meta, t.New("llvm", meta.Version, source, &PackageAttr{
Exclusive: true,
Bin: []string{
"chmod",
"mkdir",
@@ -187,6 +193,10 @@ func init() {
"tr",
"awk",
},
ScriptEarly: `
rm -f /system/bin/clang++.cfg
`,
}, &CMakeHelper{
Append: []string{"llvm"},
@@ -196,6 +206,10 @@ ln -s ld.lld /work/system/bin/ld
ln -s clang /work/system/bin/cc
ln -s clang /work/system/bin/cpp
ln -s clang++ /work/system/bin/c++
cat << EOF > /work/system/bin/clang++.cfg
-static-libstdc++
-Wl,--exclude-libs,libunwind.a:libc++.a:libc++experimental.a:libc++abi.a
EOF
`,
// LLVM_LINK_LLVM_DYLIB causes llvm test suite to leak system
+7 -6
View File
@@ -52,15 +52,16 @@ type MakeHelper struct {
var _ Helper = new(MakeHelper)
// extra returns make and other optional dependencies.
func (attr *MakeHelper) extra(flag int) P {
extra := P{_make}
if (attr == nil || !attr.OmitDefaults) && flag&TEarly == 0 {
extra = append(extra,
func (attr *MakeHelper) extra() P {
if attr == nil || !attr.OmitDefaults {
return P{
_make,
_awk,
_coreutils,
)
}
}
return extra
return P{_make}
}
// wantsChmod returns whether the build system needs to be generated.
+1 -1
View File
@@ -35,7 +35,7 @@ type MesonHelper struct {
var _ Helper = new(MesonHelper)
// extra returns hardcoded meson runtime dependencies.
func (attr *MesonHelper) extra(int) P {
func (attr *MesonHelper) extra() P {
if !attr.isPython() {
return P{_muon}
}
+2 -1
View File
@@ -10,7 +10,9 @@ package bison {
compress = gzip;
};
toyboxEarly = true;
exec = make {
omitDefaults = true;
check = [
"TESTSUITEFLAGS=" + jobsFlagE + "' " + skipGNUTests {
tests = [
@@ -24,6 +26,5 @@ package bison {
];
};
toyboxEarly = true;
inputs = [ m4 ];
}
+3 -1
View File
@@ -18,5 +18,7 @@ package byacc {
writable = true;
chmod = true;
exec = make {};
exec = make {
omitDefaults = true;
};
}
+12 -4
View File
@@ -17,9 +17,11 @@ package m4 {
chmod +w tests/test-c32ispunct.sh && echo '#!/bin/sh' > tests/test-c32ispunct.sh
`;
exec = make {};
toyboxEarly = true;
exec = make {
omitDefaults = true;
};
inputs = [
diffutils,
@@ -194,7 +196,9 @@ test_disable '#!/bin/sh' tests/cmp
`;
toyboxEarly = true;
exec = make {};
exec = make {
omitDefaults = true;
};
}
package patch {
@@ -218,7 +222,9 @@ test_disable '#!/bin/sh' tests/need-filename
`;
toyboxEarly = true;
exec = make {};
exec = make {
omitDefaults = true;
};
}
package grep {
@@ -328,6 +334,7 @@ package bash {
toyboxEarly = true;
exec = make {
omitDefaults = true;
configure = {
"without-bash-malloc";
};
@@ -364,6 +371,7 @@ test_disable 'int main(){return 0;}' gnulib-tests/test-lchown.c
`;
exec = make {
omitDefaults = true;
configure = {
"enable-single-binary": "symlinks";
};
+1
View File
@@ -25,6 +25,7 @@ package kernel-headers {
toyboxEarly = true;
exec = make {
omitDefaults = true;
skipConfigure = true;
make = [
"-f /usr/src/kernel-headers/Makefile",
+1
View File
@@ -15,6 +15,7 @@ package rsync {
toyboxEarly = true;
exec = make {
omitDefaults = true;
configure = {
"disable-openssl";
"disable-xxhash";
+1 -1
View File
@@ -82,7 +82,7 @@ package spirv-llvm-translator {
};
// litArgs emits shell syntax
early = "\nexport LIT_OPTS=" + litArgs {
early = "rm -f /system/bin/clang++.cfg\nexport LIT_OPTS=" + litArgs {
verbose = true;
skipChecks = arch {
+1 -1
View File
@@ -9,7 +9,7 @@ type MakeMakerHelper struct {
}
// extra returns perl.
func (*MakeMakerHelper) extra(int) P { return P{_perl, _make} }
func (*MakeMakerHelper) extra() P { return P{_perl, _make} }
// wantsChmod returns true.
func (*MakeMakerHelper) wantsChmod() bool { return true }
+1 -1
View File
@@ -26,7 +26,7 @@ type PipHelper struct {
var _ Helper = new(PipHelper)
// extra returns python, or pytest if defaults are assumed.
func (attr *PipHelper) extra(int) P {
func (attr *PipHelper) extra() P {
if attr == nil || (!attr.SkipCheck && attr.Check == "") {
return P{_pythonPyTest}
}
+127 -134
View File
@@ -121,6 +121,12 @@ const (
// Std denotes the standard Rosa OS toolchain.
Std
// Stage3 denotes the stage3 Rosa OS toolchain built on [Std]. Software
// built on this toolchain should be identical to [Std]. This is generally
// for validation of LLVM 3-stage nondeterminism only, due to dynamic
// linking complications.
Stage3
// _stageEnd is the total number of stages available and does not denote a
// valid toolchain.
_stageEnd
@@ -149,7 +155,7 @@ func (t Stage) isIntermediate() bool {
// isStd returns whether t is considered functionally equivalent to [Std].
func (t Stage) isStd() bool {
switch t {
case stageStdGentoo, Std:
case stageStdGentoo, Std, Stage3:
return true
default:
return false
@@ -198,17 +204,6 @@ const scriptName = "all"
// build script under.
var absCureScript = AbsSystem.Append(scriptName)
const (
// TExclusive denotes an exclusive [pkg.Artifact].
TExclusive = 1 << iota
// TEarly hints for an early variant of toybox to be used when available.
TEarly
// TNoToolchain excludes the LLVM toolchain.
TNoToolchain
// THostNet arranges for a [pkg.KindExecNet] to be created.
THostNet
)
var (
_stage0Dist = H("stage0-dist")
_mksh = H("mksh")
@@ -218,102 +213,6 @@ var (
_patch = H("patch")
)
// New returns a [pkg.Artifact] based on a [Toolchain] via s.
func (t Toolchain) New(
name string,
flag int,
extra []pkg.Artifact,
knownChecksum *pkg.Checksum,
env []string,
script string,
paths ...pkg.ExecPath,
) pkg.Artifact {
const lcMessages = "LC_MESSAGES=C.UTF-8"
var support []pkg.Artifact
switch t.stage {
case stageGentoo, stageEarly:
name += "-boot"
support = append(support, extra...)
support = append(support, NewEtc(true))
if t.stage == stageEarly {
_, a := t.MustLoad(_stage0Dist)
support = append(support, a)
} else if t.gentooStage3 == nil {
panic(os.ErrInvalid)
} else {
support = append(support,
t.gentooStage3,
gentooOverlay{t.gentooStage3},
)
env = append(env,
"CC=clang",
"CXX=clang++",
)
}
env = fixupEnviron(env, []string{
EnvTriple + "=" + t.triple(),
lcMessages,
"LDFLAGS=" + t.earlyLDFLAGS(true),
}, "/system/bin",
"/usr/bin",
)
case stageIntermediateGentoo, stageStdGentoo,
stageIntermediate, Std:
if t.stage.isIntermediate() {
name += "-std"
}
toybox := _toybox
if flag&TEarly != 0 {
toybox = _toyboxEarly
}
base := _llvm
if flag&TNoToolchain != 0 {
base = _musl
}
support = slices.Concat(extra, t.S.New(t.stage-1).Append([]pkg.Artifact{
NewEtc(false),
},
base,
_mksh,
toybox,
))
env = fixupEnviron(env, []string{
EnvTriple + "=" + t.triple(),
lcMessages,
}, "/system/bin", "/bin")
default:
panic("unsupported toolchain " + strconv.Itoa(int(t.stage)))
}
return pkg.NewExec(
name, t.arch, knownChecksum, pkg.ExecTimeoutMax,
flag&THostNet != 0,
flag&TExclusive != 0,
fhs.AbsRoot, env,
absCureScript,
nil,
slices.Concat([]pkg.ExecPath{pkg.Path(
fhs.AbsRoot, true,
support...,
), pkg.Path(
absCureScript, false,
pkg.NewFile(scriptName, []byte(
"#!/system/bin/sh\n"+
"set -eu -o pipefail\n"+
script,
)),
)}, paths)...,
)
}
// HasStageEarly returns whether a stage0 distribution is available.
func (s *S) HasStageEarly() (ok bool) {
func() {
@@ -363,9 +262,14 @@ function apply {
buf.WriteString("'\n")
}
}
return t.New(name+"-src", 0, t.Append(nil,
return t.New(name+"-src", Unversioned, nil, &PackageAttr{
Paths: paths,
}, &GenericHelper{
Build: buf.String(),
},
_patch,
), nil, nil, buf.String(), paths...)
)
}
// helperInPlace is a special directory value for omitting the cd statement.
@@ -374,7 +278,7 @@ const helperInPlace = "\x00"
// Helper is a build system helper for [Toolchain.NewPackage].
type Helper interface {
// extra returns helper-specific dependencies.
extra(flag int) P
extra() P
// wantsChmod returns whether the source directory should be made writable.
wantsChmod() bool
@@ -416,12 +320,19 @@ type PackageAttr struct {
// Whether to replace /usr/bin/ with a symlink to /bin/.
PopulateUsrBin bool
// Hint for an early variant of toybox to be used when available.
Early bool
// Exclude the LLVM toolchain.
NoToolchain bool
// Whether the resulting [pkg.Artifact] is exclusive.
Exclusive bool
// Whether to create [pkg.KindExecNet] instead of [pkg.KindExec].
HostNet bool
// Unregistered extras.
Extra []pkg.Artifact
// Passed through to [Toolchain.New], before source.
Paths []pkg.ExecPath
// Passed through to [Toolchain.New].
Flag int
}
// pa holds whether an [ArtifactH] is present.
@@ -462,8 +373,8 @@ func (t Toolchain) Append(a []pkg.Artifact, handles ...ArtifactH) []pkg.Artifact
return a
}
// NewPackage constructs a [pkg.Artifact] via a build system helper.
func (t Toolchain) NewPackage(
// New constructs a [pkg.Artifact] via a build system helper.
func (t Toolchain) New(
name, version string,
source pkg.Artifact,
attr *PackageAttr,
@@ -481,25 +392,96 @@ func (t Toolchain) NewPackage(
if version != Unversioned {
rn = name + "-" + version
}
root := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra))
root = append(root, attr.Extra...)
const lcMessages = "LC_MESSAGES=C.UTF-8"
srn := rn
env := slices.Clone(attr.Env)
var extraBoot []ArtifactH
switch t.stage {
case stageGentoo, stageEarly:
srn += "-boot"
root = append(root, NewEtc(true))
if t.stage == stageEarly {
extraBoot = append(extraBoot, _stage0Dist)
} else if t.gentooStage3 == nil {
panic(os.ErrInvalid)
} else {
env = append(env,
"CC=clang",
"CXX=clang++",
)
}
env = fixupEnviron(env, []string{
EnvTriple + "=" + t.triple(),
lcMessages,
"LDFLAGS=" + t.earlyLDFLAGS(true),
}, "/system/bin",
"/usr/bin",
)
case stageIntermediateGentoo, stageStdGentoo,
stageIntermediate, Std, Stage3:
if t.stage.isIntermediate() {
srn += "-std"
}
if t.stage == Stage3 {
srn += "-stage4"
}
toybox := _toybox
if attr.Early {
toybox = _toyboxEarly
}
base := _llvm
if attr.NoToolchain {
base = _musl
}
root = append(root, NewEtc(false))
extraBoot = append(extraBoot,
base,
_mksh,
toybox,
)
env = fixupEnviron(env, []string{
EnvTriple + "=" + t.triple(),
lcMessages,
}, "/system/bin", "/bin")
default:
panic("unsupported toolchain " + strconv.Itoa(int(t.stage)))
}
wantsChmod, wantsWrite := helper.wantsChmod(), helper.wantsWrite()
extraRes := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra))
extraRes = append(extraRes, attr.Extra...)
{
pv := paGet()
for _, p := range helper.extra(attr.Flag) {
extraRes = t.appendHandle(extraRes, pv, p)
for _, p := range helper.extra() {
root = t.appendHandle(root, pv, p)
}
for _, p := range extra {
extraRes = t.appendHandle(extraRes, pv, p)
root = t.appendHandle(root, pv, p)
}
boot := t.S.New(t.stage - 1)
for _, p := range extraBoot {
root = boot.appendHandle(root, pv, p)
}
paPut(pv)
}
var (
scriptEarly string
sourceSuffix string
)
if t.stage == stageGentoo {
root = append(root,
t.gentooStage3,
gentooOverlay{t.gentooStage3},
)
}
var scriptEarly string
dir, create := helper.wantsDir()
helperScriptEarly := helper.scriptEarly()
if attr.EnterSource ||
@@ -524,7 +506,7 @@ cd '/usr/src/` + name + `/'
if source != nil {
paths = slices.Concat(attr.Paths, []pkg.ExecPath{
pkg.Path(AbsUsrSrc.Append(
name+sourceSuffix,
name,
), attr.Writable || wantsWrite, t.NewPatchedSource(
rn, source, !attr.Chmod && !wantsChmod, attr.Patches...,
)),
@@ -551,14 +533,25 @@ ln -s ../bin /usr/`
scriptEarly += " /bin/)\n"
}
return t.New(
rn,
attr.Flag,
extraRes,
attr.KnownChecksum,
attr.Env,
scriptEarly+helper.script(t, name),
paths...,
return pkg.NewExec(
srn, t.arch, attr.KnownChecksum, pkg.ExecTimeoutMax,
attr.HostNet,
attr.Exclusive,
fhs.AbsRoot, env,
absCureScript,
nil,
slices.Concat([]pkg.ExecPath{pkg.Path(
fhs.AbsRoot, true,
root...,
), pkg.Path(
absCureScript, false,
pkg.NewFile(scriptName, []byte(
"#!/system/bin/sh\n"+
"set -eu -o pipefail\n"+
scriptEarly+helper.script(t, name),
)),
)}, paths)...,
)
}
@@ -576,7 +569,7 @@ type GenericHelper struct {
var _ Helper = new(GenericHelper)
// extra is a noop.
func (*GenericHelper) extra(int) P { return nil }
func (*GenericHelper) extra() P { return nil }
// wantsChmod returns false.
func (*GenericHelper) wantsChmod() bool { return false }
+6 -19
View File
@@ -989,9 +989,6 @@ func (ctx *evalContext) pf(
attr PackageAttr
patches []string
files []KV
excl bool
early bool
minimal bool
anitya int64
kc any
@@ -1020,9 +1017,9 @@ func (ctx *evalContext) pf(
k("populateUsrBin"): &attr.PopulateUsrBin,
k("patches"): &patches,
k("files"): &files,
k("exclusive"): &excl,
k("toyboxEarly"): &early,
k("minimal"): &minimal,
k("exclusive"): &attr.Exclusive,
k("toyboxEarly"): &attr.Early,
k("minimal"): &attr.NoToolchain,
k("checksum"): &kc,
k("output"): &output,
@@ -1116,16 +1113,6 @@ func (ctx *evalContext) pf(
)
}
if excl {
attr.Flag |= TExclusive
}
if early {
attr.Flag |= TEarly
}
if minimal {
attr.Flag |= TNoToolchain
}
if kc != nil {
checksum, ok := kc.(string)
if !ok {
@@ -1162,7 +1149,7 @@ func (ctx *evalContext) pf(
}
}
v = cachedArtifact{&meta, ctx.t.NewPackage(
v = cachedArtifact{&meta, ctx.t.New(
meta.Name,
meta.Version,
source,
@@ -1175,8 +1162,8 @@ func (ctx *evalContext) pf(
}
var (
// ErrToplevel is returned by [S.Evaluate] when encountering a toplevel
// expression other than a package declaration.
// ErrToplevel is returned by [S.RegisterAzalea] when encountering a
// toplevel expression other than a package declaration.
ErrToplevel = errors.New("top level must only contain package declarations")
)