cmd/mbf: package status dashboard

This displays package metadata with optional status from a report.

.gitignore

@@ -7,7 +7,7 @@
 
 # go generate
 /cmd/hakurei/LICENSE
-/cmd/pkgserver/ui/static/*.js
+/cmd/mbf/internal/pkgserver/ui/static
 /internal/pkg/testdata/testtool
 /internal/rosa/hakurei_current.tar.gz
 

cmd/mbf/info.go

@@ -17,25 +17,12 @@ func commandInfo(
 	args []string,
 	w io.Writer,
 	writeStatus bool,
-	reportPath string,
+	r *rosa.Report,
 ) (err error) {
 	if len(args) == 0 {
 		return errors.New("info requires at least 1 argument")
 	}
 
-	var r *rosa.Report
-	if reportPath != "" {
-		if r, err = rosa.OpenReport(reportPath); err != nil {
-			return err
-		}
-		defer func() {
-			if closeErr := r.Close(); err == nil {
-				err = closeErr
-			}
-		}()
-		defer r.HandleAccess(&err)()
-	}
-
 	// recovered by HandleAccess
 	mustPrintln := func(a ...any) {
 		if _, _err := fmt.Fprintln(w, a...); _err != nil {

cmd/mbf/info_test.go

@@ -95,7 +95,7 @@ status      : not in report
 			var (
 				cm  *cache
 				buf strings.Builder
-				rp  string
+				r   *rosa.Report
 			)
 
 			if tc.status != nil || tc.report != "" {
@@ -108,14 +108,25 @@ status      : not in report
 			}
 
 			if tc.report != "" {
-				rp = filepath.Join(t.TempDir(), "report")
+				pathname := filepath.Join(t.TempDir(), "report")
-				if err := os.WriteFile(
+				err := os.WriteFile(
-					rp,
+					pathname,
 					unsafe.Slice(unsafe.StringData(tc.report), len(tc.report)),
 					0400,
-				); err != nil {
+				)
+				if err != nil {
 					t.Fatal(err)
 				}
+
+				r, err = rosa.OpenReport(pathname)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer func() {
+					if err = r.Close(); err != nil {
+						t.Fatal(err)
+					}
+				}()
 			}
 
 			if tc.status != nil {
@@ -157,7 +168,7 @@ status      : not in report
 				tc.args,
 				&buf,
 				cm != nil,
-				rp,
+				r,
 			); !reflect.DeepEqual(err, wantErr) {
 				t.Fatalf("commandInfo: error = %v, want %v", err, wantErr)
 			}

cmd/mbf/internal/pkgserver/api.go

@@ -1,6 +1,8 @@
-package main
+// Package pkgserver implements the package metadata service backend.
+package pkgserver
 
 import (
+	"context"
 	"encoding/json"
 	"log"
 	"net/http"
@@ -8,6 +10,7 @@ import (
 	"path"
 	"strconv"
 	"sync"
+	"time"
 
 	"hakurei.app/internal/info"
 	"hakurei.app/internal/rosa"
@@ -158,6 +161,29 @@ func (index *packageIndex) registerAPI(mux *http.ServeMux) {
 	mux.HandleFunc("GET /status/", index.newStatusHandler(true))
 }
 
+// Register arranges for mux to service API requests.
+func Register(ctx context.Context, mux *http.ServeMux, report *rosa.Report) error {
+	var index packageIndex
+	index.search = make(searchCache)
+	if err := index.populate(report); err != nil {
+		return err
+	}
+	ticker := time.NewTicker(1 * time.Minute)
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				ticker.Stop()
+				return
+			case <-ticker.C:
+				index.search.clean()
+			}
+		}
+	}()
+	index.registerAPI(mux)
+	return nil
+}
+
 // writeAPIPayload sets headers common to API responses and encodes payload as
 // JSON for the response body.
 func writeAPIPayload(w http.ResponseWriter, payload any) {

cmd/mbf/internal/pkgserver/api_test.go

@@ -1,4 +1,4 @@
-package main
+package pkgserver
 
 import (
 	"net/http"
@@ -118,11 +118,9 @@ func TestAPIGet(t *testing.T) {
 			checkStatus(t, resp, http.StatusOK)
 			checkAPIHeader(t, w.Header())
 			checkPayloadFunc(t, resp, func(got *struct {
-				Count  int         `json:"count"`
 				Values []*metadata `json:"values"`
 			}) bool {
-				return got.Count == len(want) &&
+				return slices.EqualFunc(got.Values, want, func(a, b *metadata) bool {
-					slices.EqualFunc(got.Values, want, func(a, b *metadata) bool {
 					return (a.Version == b.Version ||
 						a.Version == rosa.Unversioned ||
 						b.Version == rosa.Unversioned) &&
@@ -136,15 +134,15 @@ func TestAPIGet(t *testing.T) {
 		})
 	}
 
-	checkWithSuffix("declarationAscending", "?limit=2&index=0&sort=0", []*metadata{
+	checkWithSuffix("declarationAscending", "?limit=2&index=1&sort=0", []*metadata{
-		{
-			Metadata: rosa.GetMetadata(0),
-			Version:  rosa.Std.Version(0),
-		},
 		{
 			Metadata: rosa.GetMetadata(1),
 			Version:  rosa.Std.Version(1),
 		},
+		{
+			Metadata: rosa.GetMetadata(2),
+			Version:  rosa.Std.Version(2),
+		},
 	})
 	checkWithSuffix("declarationAscending offset", "?limit=3&index=5&sort=0", []*metadata{
 		{

cmd/mbf/internal/pkgserver/index.go

@@ -1,4 +1,4 @@
-package main
+package pkgserver
 
 import (
 	"cmp"
@@ -50,7 +50,7 @@ type metadata struct {
 }
 
 // populate deterministically populates packageIndex, optionally with a report.
-func (index *packageIndex) populate(cache *pkg.Cache, report *rosa.Report) (err error) {
+func (index *packageIndex) populate(report *rosa.Report) (err error) {
 	if report != nil {
 		defer report.HandleAccess(&err)()
 		index.handleAccess = report.HandleAccess
@@ -58,6 +58,7 @@ func (index *packageIndex) populate(cache *pkg.Cache, report *rosa.Report) (err
 
 	var work [rosa.PresetUnexportedStart]*metadata
 	index.names = make(map[string]*metadata)
+	ir := pkg.NewIR()
 	for p := range rosa.PresetUnexportedStart {
 		m := metadata{
 			p: p,
@@ -72,8 +73,8 @@ func (index *packageIndex) populate(cache *pkg.Cache, report *rosa.Report) (err
 			m.Version = ""
 		}
 
-		if cache != nil && report != nil {
+		if report != nil {
-			id := cache.Ident(rosa.Std.Load(p))
+			id := ir.Ident(rosa.Std.Load(p))
 			m.ids = pkg.Encode(id.Value())
 			m.status, m.Size = report.ArtifactOf(id)
 			m.HasReport = m.Size >= 0

cmd/mbf/internal/pkgserver/index_test.go

@@ -1,4 +1,4 @@
-package main
+package pkgserver
 
 import (
 	"bytes"
@@ -15,7 +15,7 @@ func newIndex(t *testing.T) *packageIndex {
 	t.Helper()
 
 	var index packageIndex
-	if err := index.populate(nil, nil); err != nil {
+	if err := index.populate(nil); err != nil {
 		t.Fatalf("populate: error = %v", err)
 	}
 	return &index

cmd/mbf/internal/pkgserver/search.go

@@ -1,4 +1,4 @@
-package main
+package pkgserver
 
 import (
 	"cmp"

cmd/mbf/internal/pkgserver/ui/index.html

@@ -3,9 +3,9 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <link rel="stylesheet" href="static/style.css">
+    <link rel="stylesheet" href="style.css">
     <title>Hakurei PkgServer</title>
-    <script src="static/index.js"></script>
+    <script src="index.js"></script>
 </head>
 <body>
 <h1>Hakurei PkgServer</h1>

cmd/mbf/internal/pkgserver/ui/ui.go

@@ -0,0 +1,9 @@
+// Package ui holds the static web UI.
+package ui
+
+import "net/http"
+
+// Register arranges for mux to serve the embedded frontend.
+func Register(mux *http.ServeMux) {
+	mux.Handle("GET /", http.FileServer(http.FS(static)))
+}

cmd/mbf/internal/pkgserver/ui/ui_full.go

@@ -0,0 +1,21 @@
+//go:build frontend
+
+package ui
+
+import (
+	"embed"
+	"io/fs"
+)
+
+//go:generate tsc
+//go:generate cp index.html style.css static
+//go:embed static
+var _static embed.FS
+
+var static = func() fs.FS {
+	if f, err := fs.Sub(_static, "static"); err != nil {
+		panic(err)
+	} else {
+		return f
+	}
+}()

cmd/mbf/internal/pkgserver/ui/ui_stub.go

@@ -1,7 +1,7 @@
 //go:build !frontend
 
-package main
+package ui
 
 import "testing/fstest"
 
-var content fstest.MapFS
+var static fstest.MapFS

cmd/mbf/main.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"log"
 	"net"
+	"net/http"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -41,6 +42,9 @@ import (
 	"hakurei.app/internal/pkg"
 	"hakurei.app/internal/rosa"
 	"hakurei.app/message"
+
+	"hakurei.app/cmd/mbf/internal/pkgserver"
+	"hakurei.app/cmd/mbf/internal/pkgserver/ui"
 )
 
 func main() {
@@ -136,7 +140,17 @@ func main() {
 	c.NewCommand(
 		"checksum", "Compute checksum of data read from standard input",
 		func([]string) error {
-			go func() { <-ctx.Done(); os.Exit(1) }()
+			done := make(chan struct{})
+			defer close(done)
+			go func() {
+				select {
+				case <-ctx.Done():
+					os.Exit(1)
+				case <-done:
+					return
+				}
+			}()
+
 			h := sha512.New384()
 			if _, err := io.Copy(h, os.Stdin); err != nil {
 				return err
@@ -170,6 +184,7 @@ func main() {
 
 	{
 		var (
+			flagBind   string
 			flagStatus bool
 			flagReport string
 		)
@@ -177,8 +192,52 @@ func main() {
 			"info",
 			"Display out-of-band metadata of an artifact",
 			func(args []string) (err error) {
-				return commandInfo(&cm, args, os.Stdout, flagStatus, flagReport)
+				const shutdownTimeout = 15 * time.Second
+
+				var r *rosa.Report
+				if flagReport != "" {
+					if r, err = rosa.OpenReport(flagReport); err != nil {
+						return err
+					}
+					defer func() {
+						if closeErr := r.Close(); err == nil {
+							err = closeErr
+						}
+					}()
+					defer r.HandleAccess(&err)()
+				}
+
+				if flagBind == "" {
+					return commandInfo(&cm, args, os.Stdout, flagStatus, r)
+				}
+
+				var mux http.ServeMux
+				ui.Register(&mux)
+				if err = pkgserver.Register(ctx, &mux, r); err != nil {
+					return
+				}
+
+				server := http.Server{Addr: flagBind, Handler: &mux}
+				go func() {
+					<-ctx.Done()
+					cc, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+					defer cancel()
+					if _err := server.Shutdown(cc); _err != nil {
+						log.Fatal(_err)
+					}
+				}()
+
+				msg.Verbosef("listening on %q", flagBind)
+				err = server.ListenAndServe()
+				if errors.Is(err, http.ErrServerClosed) {
+					err = nil
+				}
+				return
 			},
+		).Flag(
+			&flagBind,
+			"bind", command.StringFlag(""),
+			"TCP address for the server to listen on",
 		).Flag(
 			&flagStatus,
 			"status", command.BoolFlag(false),

cmd/mbf/main_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"net"
+	"os"
+	"testing"
+
+	"hakurei.app/internal/rosa"
+)
+
+func TestMain(m *testing.M) {
+	rosa.DropCaches(rosa.OptLLVMNoLTO)
+	os.Exit(m.Run())
+}
+
+func TestCureAll(t *testing.T) {
+	t.Parallel()
+	const env = "ROSA_TEST_DAEMON"
+
+	if !testing.Verbose() {
+		t.Skip("verbose flag not set")
+	}
+
+	pathname, ok := os.LookupEnv(env)
+	if !ok {
+		t.Skip(env + " not set")
+	}
+
+	addr := net.UnixAddr{Net: "unix", Name: pathname}
+	t.Cleanup(func() {
+		if t.Failed() {
+			if err := abortRemote(t.Context(), &addr, false); err != nil {
+				t.Fatal(err)
+			}
+		}
+	})
+
+	for i := range rosa.PresetEnd {
+		p := rosa.PArtifact(i)
+		t.Run(rosa.GetMetadata(p).Name, func(t *testing.T) {
+			_, err := cureRemote(t.Context(), &addr, rosa.Std.Load(p), 0)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}

cmd/pkgserver/main.go

@@ -1,114 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"log"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"hakurei.app/check"
-	"hakurei.app/command"
-	"hakurei.app/internal/pkg"
-	"hakurei.app/internal/rosa"
-	"hakurei.app/message"
-)
-
-const shutdownTimeout = 15 * time.Second
-
-func main() {
-	log.SetFlags(0)
-	log.SetPrefix("pkgserver: ")
-
-	var (
-		flagBaseDir string
-		flagAddr    string
-	)
-
-	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
-	defer stop()
-	msg := message.New(log.Default())
-
-	c := command.New(os.Stderr, log.Printf, "pkgserver", func(args []string) error {
-		var (
-			cache  *pkg.Cache
-			report *rosa.Report
-		)
-		switch len(args) {
-		case 0:
-			break
-
-		case 1:
-			baseDir, err := check.NewAbs(flagBaseDir)
-			if err != nil {
-				return err
-			}
-
-			cache, err = pkg.Open(ctx, msg, 0, 0, 0, baseDir)
-			if err != nil {
-				return err
-			}
-			defer cache.Close()
-
-			report, err = rosa.OpenReport(args[0])
-			if err != nil {
-				return err
-			}
-
-		default:
-			return errors.New("pkgserver requires 1 argument")
-
-		}
-
-		var index packageIndex
-		index.search = make(searchCache)
-		if err := index.populate(cache, report); err != nil {
-			return err
-		}
-		ticker := time.NewTicker(1 * time.Minute)
-		go func() {
-			for {
-				select {
-				case <-ctx.Done():
-					ticker.Stop()
-					return
-				case <-ticker.C:
-					index.search.clean()
-				}
-			}
-		}()
-		var mux http.ServeMux
-		uiRoutes(&mux)
-		index.registerAPI(&mux)
-		server := http.Server{
-			Addr:    flagAddr,
-			Handler: &mux,
-		}
-		go func() {
-			<-ctx.Done()
-			c, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
-			defer cancel()
-			if err := server.Shutdown(c); err != nil {
-				log.Fatal(err)
-			}
-		}()
-		return server.ListenAndServe()
-	}).Flag(
-		&flagBaseDir,
-		"b", command.StringFlag(""),
-		"base directory for cache",
-	).Flag(
-		&flagAddr,
-		"addr", command.StringFlag(":8067"),
-		"TCP network address to listen on",
-	)
-	c.MustParse(os.Args[1:], func(err error) {
-		if errors.Is(err, http.ErrServerClosed) {
-			os.Exit(0)
-		}
-		log.Fatal(err)
-	})
-}

cmd/pkgserver/ui.go

@@ -1,33 +0,0 @@
-package main
-
-import "net/http"
-
-func serveWebUI(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
-	w.Header().Set("Pragma", "no-cache")
-	w.Header().Set("Expires", "0")
-	w.Header().Set("X-Content-Type-Options", "nosniff")
-	w.Header().Set("X-XSS-Protection", "1")
-	w.Header().Set("X-Frame-Options", "DENY")
-
-	http.ServeFileFS(w, r, content, "ui/index.html")
-}
-func serveStaticContent(w http.ResponseWriter, r *http.Request) {
-	switch r.URL.Path {
-	case "/static/style.css":
-		http.ServeFileFS(w, r, content, "ui/static/style.css")
-	case "/favicon.ico":
-		http.ServeFileFS(w, r, content, "ui/static/favicon.ico")
-	case "/static/index.js":
-		http.ServeFileFS(w, r, content, "ui/static/index.js")
-	default:
-		http.NotFound(w, r)
-
-	}
-}
-
-func uiRoutes(mux *http.ServeMux) {
-	mux.HandleFunc("GET /{$}", serveWebUI)
-	mux.HandleFunc("GET /favicon.ico", serveStaticContent)
-	mux.HandleFunc("GET /static/", serveStaticContent)
-}

cmd/pkgserver/ui_full.go

@@ -1,9 +0,0 @@
-//go:build frontend
-
-package main
-
-import "embed"
-
-//go:generate tsc -p ui
-//go:embed ui/*
-var content embed.FS

internal/rosa/gnu.go

@@ -2,10 +2,47 @@ package rosa
 
 import (
 	"runtime"
+	"slices"
+	"strconv"
+	"strings"
 
 	"hakurei.app/internal/pkg"
 )
 
+// skipGNUTests generates a string for skipping specific tests by number in a
+// GNU test suite. This is nontrivial because the test suite does not support
+// excluding tests in any way, so ranges for all but the skipped tests have to
+// be specified instead.
+//
+// For example, to skip test 764, ranges around the skipped test must be
+// specified:
+//
+//	1-763 765-
+//
+// Tests are numbered starting from 1. The resulting string is unquoted.
+func skipGNUTests(tests ...int) string {
+	tests = slices.Clone(tests)
+	slices.Sort(tests)
+
+	var buf strings.Builder
+
+	if tests[0] != 1 {
+		buf.WriteString("1-")
+	}
+
+	for i, n := range tests {
+		if n != 1 && (i == 0 || tests[i-1] != n-1) {
+			buf.WriteString(strconv.Itoa(n - 1))
+			buf.WriteString(" ")
+		}
+		if i == len(tests)-1 || tests[i+1] != n+1 {
+			buf.WriteString(strconv.Itoa(n + 1))
+			buf.WriteString("-")
+		}
+	}
+	return buf.String()
+}
+
 func (t Toolchain) newM4() (pkg.Artifact, string) {
 	const (
 		version  = "1.4.21"
@@ -47,7 +84,15 @@ func (t Toolchain) newBison() (pkg.Artifact, string) {
 		"https://ftpmirror.gnu.org/gnu/bison/bison-"+version+".tar.gz",
 		checksum,
 		pkg.TarGzip,
-	), nil, (*MakeHelper)(nil),
+	), nil, &MakeHelper{
+		Check: []string{
+			"TESTSUITEFLAGS=" + jobsFlagE + "' " + skipGNUTests(
+				// clang miscompiles (SIGILL)
+				764,
+			) + "'",
+			"check",
+		},
+	},
 		M4,
 		Diffutils,
 		Sed,

internal/rosa/gnu_test.go

@@ -0,0 +1,35 @@
+package rosa
+
+import (
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestSkipGNUTests(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		tests []int
+		want  string
+	}{
+		{[]int{764}, "1-763 765-"},
+		{[]int{764, 0xcafe, 37, 9}, "1-8 10-36 38-763 765-51965 51967-"},
+		{[]int{1, 2, 0xbed}, "3-3052 3054-"},
+		{[]int{3, 4}, "1-2 5-"},
+	}
+	for _, tc := range testCases {
+		t.Run(strings.Join(slices.Collect(func(yield func(string) bool) {
+			for _, n := range tc.tests {
+				yield(strconv.Itoa(n))
+			}
+		}), ","), func(t *testing.T) {
+			t.Parallel()
+
+			if got := skipGNUTests(tc.tests...); got != tc.want {
+				t.Errorf("skipGNUTests: %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

internal/rosa/go.go

@@ -10,9 +10,9 @@ import (
 // newGoBootstrap returns the Go bootstrap toolchain.
 func (t Toolchain) newGoBootstrap() pkg.Artifact {
 	const checksum = "8o9JL_ToiQKadCTb04nvBDkp8O1xiWOolAxVEqaTGodieNe4lOFEjlOxN3bwwe23"
-	return t.New("go1.4-bootstrap", 0, []pkg.Artifact{
+	return t.New("go1.4-bootstrap", 0, t.AppendPresets(nil,
-		t.Load(Bash),
+		Bash,
-	}, nil, []string{
+	), nil, []string{
 		"CGO_ENABLED=0",
 	}, `
 mkdir -p /var/tmp/ /work/system/
@@ -35,9 +35,9 @@ func (t Toolchain) newGo(
 	script string,
 	extra ...pkg.Artifact,
 ) pkg.Artifact {
-	return t.New("go"+version, 0, slices.Concat([]pkg.Artifact{
+	return t.New("go"+version, 0, t.AppendPresets(extra,
-		t.Load(Bash),
+		Bash,
-	}, extra), nil, slices.Concat([]string{
+	), nil, slices.Concat([]string{
 		"CC=cc",
 		"GOCACHE=/tmp/gocache",
 		"GOROOT_BOOTSTRAP=/system/go",
@@ -127,8 +127,8 @@ sed -i \
 	)
 
 	go125 := t.newGo(
-		"1.25.7",
+		"1.25.9",
-		"fyylHdBVRUobnBjYj3NKBaYPUw3kGmo2mEELiZonOYurPfbarNU1x77B99Fjut7Q",
+		"gShJb9uOMk5AxqPSwvn53ZO56S6PyP6nfojzrHUiJ3krAvrgjJpYa6-DPA-jxbpN",
 		[]string{"CGO_ENABLED=0"}, `
 sed -i \
 	's,/lib/ld-musl-`+linuxArch()+`.so.1,/system/bin/linker,' \
@@ -156,7 +156,9 @@ sed -i \
 	internal/runtime/gc/scan/scan_amd64.go
 
 rm \
-	os/root_unix_test.go
+	os/root_unix_test.go \
+	cmd/cgo/internal/testsanitizers/tsan_test.go \
+	cmd/cgo/internal/testsanitizers/cshared_test.go
 `, go125,
 	), version
 }

internal/rosa/rosa_test.go

@@ -28,6 +28,7 @@ var (
 )
 
 func TestMain(m *testing.M) {
+	rosa.DropCaches(rosa.OptLLVMNoLTO)
 	container.TryArgv0(nil)
 
 	code := m.Run()

internal/rosa/tamago.go

@@ -27,8 +27,14 @@ chmod -R +w ..
 sed -i \
 	's,/lib/ld-musl-`+linuxArch()+`.so.1,/system/bin/linker,' \
 	cmd/link/internal/`+runtime.GOARCH+`/obj.go
+sed -i \
+	's/cpu.X86.HasAVX512VBMI/& \&\& cpu.X86.HasPOPCNT/' \
+	internal/runtime/gc/scan/scan_amd64.go
+
 rm \
-	os/root_unix_test.go
+	os/root_unix_test.go \
+	cmd/cgo/internal/testsanitizers/tsan_test.go \
+	cmd/cgo/internal/testsanitizers/cshared_test.go
 
 ./all.bash
 `, pkg.Path(AbsUsrSrc.Append("tamago"), false, newFromGitHub(