package rosa import ( "context" "encoding/json" "errors" "io" "io/fs" "net/http" "path/filepath" "reflect" "runtime" "slices" "strconv" "strings" "sync" "sync/atomic" "unique" "unsafe" "hakurei.app/internal/pkg" "hakurei.app/internal/rosa/azalea" ) // ArtifactH is a handle of the unique name of a prepared [pkg.Artifact]. type ArtifactH unique.Handle[string] // H is a convenient function for acquiring an [ArtifactH] by name. func H(name string) ArtifactH { return ArtifactH(unique.Make(name)) } // String returns the name of p. func (handle ArtifactH) String() string { return unique.Handle[string](handle).Value() } // MarshalJSON represents [ArtifactH] by its [Artifact.Name]. func (handle ArtifactH) MarshalJSON() ([]byte, error) { return json.Marshal(handle.String()) } // UnmarshalJSON resolves [ArtifactH] by its [Artifact.Name]. func (handle *ArtifactH) UnmarshalJSON(data []byte) error { var name string if err := json.Unmarshal(data, &name); err != nil { return err } *handle = ArtifactH(unique.Make(name)) return nil } // A HandleError is panicked when attempting to acquire an [Artifact] via an // invalid [ArtifactH] with the Must suite of methods. type HandleError ArtifactH func (e HandleError) Error() string { return "package " + strconv.Quote(ArtifactH(e).String()) + " not available" } type ( // Stage denotes the infrastructure to compile a [pkg.Artifact] on. Stage uint32 // Toolchain refers to an instance of [S], and a [Stage] to compile on. Toolchain struct { stage Stage *S } ) // P represents multiple [ArtifactH]. type P []ArtifactH // Metadata is stage-agnostic immutable data around [Artifact] not directly // representable in the resulting [pkg.Artifact]. type Metadata struct { // Unique package name. Name string `json:"name"` // Short user-facing description. Description string `json:"description"` // Project home page. Website string `json:"website,omitempty"` // Runtime dependencies. Dependencies P `json:"dependencies"` // Package version. Version string `json:"version"` // Project identifier on [Anitya]. // // [Anitya]: https://release-monitoring.org/ ID int `json:"-"` // Whether to exclude from exported functions. Exclude bool `json:"exclude,omitempty"` // Optional custom version checking behaviour. latest func(v *Versions) string } // GetLatest returns the latest version described by v. func (meta *Metadata) GetLatest(v *Versions) string { if meta.latest != nil { return meta.latest(v) } return v.Latest } // Unversioned denotes an unversioned [Artifact]. const Unversioned = "\x00" // UnpopulatedIDError is returned by [Artifact.GetLatest] for an instance of // [Artifact] where ID is not populated. type UnpopulatedIDError struct{} func (UnpopulatedIDError) Unwrap() error { return errors.ErrUnsupported } func (UnpopulatedIDError) Error() string { return "Anitya ID is not populated" } // Versions are package versions returned by Anitya. type Versions struct { // The latest version for the project, as determined by the version sorting algorithm. Latest string `json:"latest_version"` // List of all versions that aren’t flagged as pre-release. Stable []string `json:"stable_versions"` // List of all versions stored, sorted from newest to oldest. All []string `json:"versions"` } // getStable returns the first Stable version, or Latest if that is unavailable. func (v *Versions) getStable() string { if len(v.Stable) == 0 { return v.Latest } return v.Stable[0] } // GetVersions returns versions fetched from Anitya. func (meta *Metadata) GetVersions(ctx context.Context) (*Versions, error) { if meta.ID == 0 { return nil, UnpopulatedIDError{} } var resp *http.Response if req, err := http.NewRequestWithContext( ctx, http.MethodGet, "https://release-monitoring.org/api/v2/versions/?project_id="+ strconv.Itoa(meta.ID), nil, ); err != nil { return nil, err } else { req.Header.Set("User-Agent", "Rosa/1.1") if resp, err = http.DefaultClient.Do(req); err != nil { return nil, err } } var v Versions err := json.NewDecoder(resp.Body).Decode(&v) return &v, errors.Join(err, resp.Body.Close()) } // Artifact is a lazily initialised [pkg.Artifact] with associated [Metadata]. type Artifact func(t Toolchain) (meta *Metadata, a pkg.Artifact) // A cachedArtifact caches satisfied [Artifact]. type cachedArtifact struct { meta *Metadata a pkg.Artifact } const ( // OptSkipCheck skips running all test suites. OptSkipCheck = 1 << iota // OptLLVMNoLTO disables LTO in all [LLVM] stages. OptLLVMNoLTO ) // S holds a set of [Artifact]. type S struct { // [ArtifactH] to [Artifact]. artifacts sync.Map // Size of artifacts. artifactCount atomic.Uint64 // Target architecture. arch string // For initialising arch. archOnce sync.Once // Built-in functions. frame azalea.Frame // For initialising frame. frameOnce sync.Once // Must only be accessed after a call to getFrame. spool sync.Pool // Options for [pkg.Artifact] created against [S]. opts int // Cached [pkg.Artifact]. c [_stageEnd]sync.Map // URL of a Gentoo stage3 tarball. gentooStage3 string // Expected checksum of gentooStage3. gentooStage3Checksum pkg.Checksum } // Clone returns a copy of s. func (s *S) Clone() *S { v := S{arch: s.arch} s.artifacts.Range(func(key, value any) bool { v.artifacts.Store(key, value) v.artifactCount.Add(1) return true }) return &v } // wantsArch must be called before accessing arch. func (s *S) wantsArch() { s.archOnce.Do(func() { if s.arch == "" { s.arch = runtime.GOARCH } }) } // Arch returns the target architecture. func (s *S) Arch() string { s.wantsArch(); return s.arch } // Flags returns the current preset flags. func (s *S) Flags() int { return s.opts } // DropCaches arranges for all cached [pkg.Artifact] to be freed some time after // it returns. Must not be used concurrently with any other method. func (s *S) DropCaches(targetArch string, flags int) { if targetArch == "" { targetArch = runtime.GOARCH } s.arch = targetArch s.opts = flags for i := range s.c { s.c[i].Clear() } } // get returns the named [Artifact]. func (s *S) get(handle ArtifactH) (f Artifact) { s.wantsArch() v, ok := s.artifacts.Load(handle) if ok { f = v.(Artifact) } return } // mustGet is like get, but panics if the named [Artifact] is not registered. func (s *S) mustGet(handle ArtifactH) (f Artifact) { f = s.get(handle) if f == nil { panic(HandleError(handle)) } return } // New returns a [Toolchain] for the specified [Stage]. func (s *S) New(stage Stage) Toolchain { return Toolchain{S: s, stage: stage} } // Std is a convenience method that returns a [Toolchain] for the [Std] stage. func (s *S) Std() Toolchain { return s.New(Std) } // LoadError wraps panicked errors reaching [Toolchain.Load]. type LoadError struct { // Offending artifact handle. Handle ArtifactH // Recovered error. Err error } func (e LoadError) Unwrap() error { return e.Err } func (e LoadError) Error() string { return "cannot load " + strconv.Quote(e.Handle.String()) + ": " + e.Err.Error() } // Load satisfies an [Artifact] referred to by an [ArtifactH]. func (t Toolchain) Load(handle ArtifactH) (*Metadata, pkg.Artifact) { defer func() { r := recover() if r == nil { return } err, ok := r.(error) if !ok { panic(r) } if _, ok = err.(LoadError); ok { panic(err) } panic(LoadError{handle, err}) }() t.wantsArch() e, ok := t.c[t.stage].Load(handle) if ok { r := e.(cachedArtifact) return r.meta, r.a } f := t.get(handle) if f == nil { return nil, nil } var r cachedArtifact r.meta, r.a = f(t) t.c[t.stage].Store(handle, r) return r.meta, r.a } // MustLoad is like Load, but panics if the named [Artifact] is not registered. func (t Toolchain) MustLoad(handle ArtifactH) (*Metadata, pkg.Artifact) { meta, a := t.Load(handle) if meta == nil { panic(HandleError(handle)) } return meta, a } // Register arranges for a new [Artifact] to be cured under s. It returns false // if another [Artifact] is already registered under the same name. func (s *S) Register(name string, f Artifact) bool { if name == "" { return false } p := ArtifactH(unique.Make(name)) _, ok := s.artifacts.LoadOrStore(p, f) if !ok { s.artifactCount.Add(1) } return !ok } // RegisterError is returned or panicked when attempting to register multiple // [Artifact] with the same name. type RegisterError ArtifactH func (e RegisterError) Error() string { if ArtifactH(e).String() == "" { return "attempting to register invalid name" } return "attempting to register " + strconv.Quote(ArtifactH(e).String()) + " twice" } // MustRegister is like Register, but panics if registration fails. func (s *S) MustRegister(name string, f Artifact) { if !s.Register(name, f) { panic(RegisterError(H(name))) } } // mustRegister registers an [Artifact] with the old function signature. // // Deprecated: Artifacts should be migrated to Register. func (s *S) mustRegister( f func(t Toolchain) (pkg.Artifact, string), meta *Metadata, ) { s.MustRegister(meta.Name, func(t Toolchain) (*Metadata, pkg.Artifact) { v := *meta a, version := f(t) v.Version = version return &v, a }) } // Count returns the number of [Artifact] registered to s. func (s *S) Count() int { return int(s.artifactCount.Load()) } // Collect returns all [ArtifactH] registered to s. func (s *S) Collect() (handles P) { handles = make(P, 0, s.Count()) s.artifacts.Range(func(key, _ any) bool { handles = append(handles, key.(ArtifactH)) return true }) slices.SortFunc(handles, func(a, b ArtifactH) int { return strings.Compare(a.String(), b.String()) }) return } // deferredGit is a call to Toolchain.newTagRemote from azalea. type deferredGit struct { url string tag string checksum string } // getFrame must be called before accessing s. func (s *S) getFrame() azalea.Frame { s.frameOnce.Do(func() { s.spool.New = func() any { v := make([]azalea.Frame, 1, 1<<4) v[0] = s.getFrame() return v } s.wantsArch() k := func(name string) unique.Handle[azalea.Ident] { return unique.Make(azalea.Ident(name)) } var ( identDefault = k("default") identArch = k(s.arch) ) s.frame.Val = map[unique.Handle[azalea.Ident]]any{ k("linuxArch"): s.linuxArch(), k("jobsE"): jobsE, k("jobsFlagE"): jobsFlagE, k("jobsLE"): jobsLE, k("jobsLFlagE"): jobsLFlagE, } s.frame.Func = map[unique.Handle[azalea.Ident]]azalea.F{ // library functions k("join"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var elems []string var sep string if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("elems"): &elems, k("sep"): &sep, }); err != nil { return } v = strings.Join(elems, sep) set = true return }}, // intenral/pkg built-ins k("remoteTar"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var url, checksum string var compress uint32 if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("url"): &url, k("checksum"): &checksum, k("compress"): &compress, }); err != nil { return } v = pkg.NewHTTPGetTar(nil, url, mustDecode(checksum), compress) set = true return }, V: map[unique.Handle[azalea.Ident]]any{ k("uncompressed"): uint32(pkg.TarUncompressed), k("gzip"): uint32(pkg.TarGzip), k("bzip2"): uint32(pkg.TarBzip2), }}, k("remoteFile"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var url, checksum string if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("url"): &url, k("checksum"): &checksum, }); err != nil { return } v = pkg.NewHTTPGet(nil, url, mustDecode(checksum)) set = true return }}, // state helpers k("arch"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var fallback any for _, arg := range args { switch arg.K { case identDefault: fallback = arg.V continue case identArch: return arg.V, true, nil } } return fallback, fallback != nil, nil }}, // convenience functions k("remoteGit"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var a deferredGit if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("url"): &a.url, k("tag"): &a.tag, k("checksum"): &a.checksum, }); err != nil { return } v = a set = true return }}, k("remoteGitLab"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var domain, suffix, ref, checksum string if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("domain"): &domain, k("suffix"): &suffix, k("ref"): &ref, k("checksum"): &checksum, }); err != nil { return } v = newFromGitLab(domain, suffix, ref, checksum) set = true return }}, k("remoteCPAN"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var author, name, version, checksum string if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("author"): &author, k("name"): &name, k("version"): &version, k("checksum"): &checksum, }); err != nil { return } v = newFromCPAN(author, name, version, checksum) set = true return }}, k("remoteGitHub"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var suffix, tag, checksum string if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("suffix"): &suffix, k("tag"): &tag, k("checksum"): &checksum, }); err != nil { return } v = newFromGitHub(suffix, tag, checksum) set = true return }}, // high-level helpers k("make"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var attr MakeHelper if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("omitDefaults"): &attr.OmitDefaults, k("generate"): &attr.Generate, k("preMake"): &attr.ScriptMakeEarly, k("preCheck"): &attr.ScriptCheckEarly, k("postInstall"): &attr.Script, k("inPlace"): &attr.InPlace, k("skipConfigure"): &attr.SkipConfigure, k("configureName"): &attr.ConfigureName, k("configure"): &attr.Configure, k("host"): &attr.Host, k("build"): &attr.Build, k("make"): &attr.Make, k("skipCheck"): &attr.SkipCheck, k("check"): &attr.Check, k("install"): &attr.Install, k("skipEarlyStageCheck"): &attr.SkipCheckEarly, }); err != nil { return } v = &attr set = true return }}, k("makeMaker"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var attr MakeMakerHelper if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("skipCheck"): &attr.SkipCheck, }); err != nil { return } v = &attr set = true return }}, k("pip"): {F: func( args azalea.FArgs, ) (v any, set bool, err error) { var attr PipHelper if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("append"): &attr.Append, k("buildIsolation"): &attr.BuildIsolation, k("enterSource"): &attr.EnterSource, k("install"): &attr.Install, k("skipCheck"): &attr.SkipCheck, k("check"): &attr.Check, k("postInstall"): &attr.Script, }); err != nil { return } v = &attr set = true return }}, } }) return s.frame } // getStack returns a new stack for azalea evaluation. func (s *S) getStack() []azalea.Frame { s.getFrame() return s.spool.Get().([]azalea.Frame) } // putStack returns a stack to spool. func (s *S) putStack(v []azalea.Frame) { s.spool.Put(v) } // toHandles makes handles out of an [azalea.Array] of identifiers. func toHandles(idents azalea.Array) (P, error) { handles := make(P, len(idents)) for i, p := range idents { if len(p) != 1 { return nil, azalea.EvaluationError{ Expr: p, Err: errors.New("concatenation not allowed for handles"), } } s, ok := p[0].(azalea.Ident) if !ok { return nil, azalea.EvaluationError{ Expr: p[0], Err: errors.New("identifiers expected for handles"), } } handles[i] = H(string(s)) } return handles, nil } // evalContext holds per-artifact context. type evalContext struct { // Backing filesystem. b fs.FS // Pending azalea function. expr *azalea.Func // Current toolchain. t Toolchain } // pf implements [azalea.PF]. func (ctx *evalContext) pf( name azalea.Ident, args azalea.FArgs, ) (v any, set bool, err error) { k := func(name string) unique.Handle[azalea.Ident] { return unique.Make(azalea.Ident(name)) } meta := Metadata{Name: string(name)} var ( attr PackageAttr patches []string excl bool early bool anitya int64 sourceA any helper Helper inputs, runtimes azalea.Array anityaFallback bool anityaLegacyCPAN bool ) if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ k("description"): &meta.Description, k("website"): &meta.Website, k("version"): &meta.Version, k("anitya"): &anitya, k("writable"): &attr.Writable, k("chmod"): &attr.Chmod, k("enterSource"): &attr.EnterSource, k("env"): &attr.Env, k("early"): &attr.ScriptEarly, k("patches"): &patches, k("exclusive"): &excl, k("toyboxEarly"): &early, k("source"): &sourceA, k("exec"): &helper, k("inputs"): &inputs, k("runtime"): &runtimes, k("anityaFallback"): &anityaFallback, k("anityaLegacyCPAN"): &anityaLegacyCPAN, }); err != nil { return } var inputsH P if inputsH, err = toHandles(inputs); err != nil { return } else if meta.Dependencies, err = toHandles(runtimes); err != nil { return } for _, pathname := range patches { var p []byte p, err = fs.ReadFile(ctx.b, pathname) if err != nil { return } attr.Patches = append(attr.Patches, KV{ strings.TrimSuffix(filepath.Base(pathname), ".patch"), unsafe.String(unsafe.SliceData(p), len(p)), }) } if excl { attr.Flag |= TExclusive } if early { attr.Flag |= TEarly } meta.ID = int(anitya) if anityaFallback { meta.latest = (*Versions).getStable } else if anityaLegacyCPAN { meta.latest = func(v *Versions) string { for _, s := range v.Stable { _, m, ok := strings.Cut(s, ".") if !ok { continue } if len(m) > 1 && m[0] == '0' { continue } return s } return v.Latest } } var source pkg.Artifact switch p := sourceA.(type) { case pkg.Artifact: source = p case deferredGit: source = ctx.t.newTagRemote(p.url, p.tag, p.checksum) default: panic(azalea.TypeError{ Concrete: reflect.TypeOf(sourceA), Asserted: reflect.TypeFor[pkg.Artifact](), }) } v = cachedArtifact{&meta, ctx.t.NewPackage( meta.Name, meta.Version, source, &attr, helper, inputsH..., )} set = true return } var ( // ErrToplevel is returned by [S.Evaluate] when encountering a toplevel // expression other than a package declaration. ErrToplevel = errors.New("top level must only contain package declarations") ) // RegisterAzalea registers all package declarations from r. The backing // filesystem is directly exposed to azalea pathnames. func (s *S) RegisterAzalea(r io.Reader, b fs.FS) error { var pending []*azalea.Func if expressions, err := azalea.Parse(r); err != nil { return err } else { pending = make([]*azalea.Func, len(expressions)) for i, expr := range expressions { f, ok := expr.(azalea.Func) if !ok || !f.Package { return ErrToplevel } pending[i] = &f } } for _, f := range pending { if !s.Register(string(f.Ident), func(t Toolchain) (*Metadata, pkg.Artifact) { v, set, err := azalea.Evaluate[cachedArtifact]( (&evalContext{b, f, t}).pf, s.getStack(), *f, ) if err != nil { panic(err) } else if !set { panic(errors.New("unexpected unset")) } return v.meta, v.a }) { return RegisterError(H(string(f.Ident))) } } return nil } // RegisterFS registers from azalea files discovered in fsys. A file is // evaluated if it is at the top level and its name has the suffix ".az", or it // is in a top-level directory with the exact file name "package.az". The // backing filesystem is directly exposed to azalea pathnames. func (s *S) RegisterFS(fsys fs.FS) error { dents, err := fs.ReadDir(fsys, ".") if err != nil { return err } var r fs.File for _, dent := range dents { if dent.IsDir() { var sub fs.FS sub, err = fs.Sub(fsys, dent.Name()) if err != nil { return err } r, err = sub.Open("package.az") if errors.Is(err, fs.ErrNotExist) { continue } if err != nil { return err } err = s.RegisterAzalea(r, sub) if _err := r.Close(); err == nil { err = _err } if err != nil { return err } continue } if !dent.Type().IsRegular() || !strings.HasSuffix(dent.Name(), ".az") { continue } r, err = fsys.Open(dent.Name()) if err != nil { return err } err = s.RegisterAzalea(r, fsys) if _err := r.Close(); err == nil { err = _err } if err != nil { return err } } return nil } // SetGentooStage3 sets the Gentoo stage3 tarball url and checksum. It panics // if given zero values or if these values have already been set. func (s *S) SetGentooStage3(url string, checksum pkg.Checksum) { if s.gentooStage3 != "" { panic(errors.New("attempting to set Gentoo stage3 url twice")) } if url == "" { panic(errors.New("attempting to set Gentoo stage3 url to the zero value")) } s.gentooStage3, s.gentooStage3Checksum = url, checksum s.DropCaches(s.Arch(), s.Flags()) }