TODO: consider writing tests for the test runner.

both on a test level and for the whole thing. i think the reporter or
registrar abstractions should deal with all timeouts, and just feed
elapsed time through all the functions: update() gets time for the
specific test, and finalize passes you the total time. this way you
don't need to do the same logic in every reporter, and you also give
a suggestion to reporter writers (i.e.: you in the future) to expose
test durations. actually tbh per-test isn't possible anywhere but in the
executor, especially when taking potential future parallel execution
into account

on the topic of parallelism: per-test is wall clock for that test,
regardless of perceived time, because no other number is useful. whole
thing is wall clock too, not cpu time

remember:
  - use monotonic clocks!! we need elapsed time, not absolute time
  - format them to more readable strings like “15h 12m” instead of
    “54738 seconds”. once things get large we can be less precise

for the go reporter: ask ozy if the go one already measures it. if so
then don't even bother serializing it

for the stream reporter: the live feed should include per-test time in
brackets or something. the final tree should only include timeout for
outliers on the long side (just shove a box plot-esque algo on it), and
if a flag is given print it for all nodes, and if another flag is given
print the n longest tests. the total time should be in the summary line
at the end in brackets à la pytest

for the dom reporter, we do the same as with the stream reporter's
outlier detection, and have a checkbox or button to dynamically
show/hide all timeouts, and another button to toggle a widget of sorts
that shows up right above the result tree which includes the n longest
tests. all these buttons should be on the same line as the summary
(successes/failures/skips). the total time should be included in the
“execution finished” text form the previous commit, i.e. “execution
finished in 15s”

~april+may

cmd/mbf/internal/pkgserver/ui/all_tests.ts

@@ -1,2 +1,2 @@
 // Import all test files to register their test suites.
-import "./sample_test.js";
+import "./index_test.js";

cmd/mbf/internal/pkgserver/ui/index_test.ts

@@ -0,0 +1,2 @@
+import { suite, test } from "./jstest/jstest.js";
+import "./index.js";

cmd/mbf/internal/pkgserver/ui/jstest/failure-closed.svg

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<!-- See failure-open.svg for an explanation of the view box dimensions. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="20" viewBox="-20,-20 160 130">
+    <!-- This triangle should match success-closed.svg, fill and stroke color notwithstanding. -->
+    <polygon points="0,0 100,50 0,100" fill="red" stroke="red" stroke-width="15" stroke-linejoin="round"/>
+    <!--
+     ! y-coordinates go before x-coordinates here to highlight the difference
+     ! (or, lack thereof) between these numbers and the ones in failure-open.svg;
+     ! try a textual diff. Make sure to keep the numbers in sync!
+     -->
+    <line y1="30" x1="10" y2="70" x2="50" stroke="white" stroke-width="16"/>
+    <line y1="30" x1="50" y2="70" x2="10" stroke="white" stroke-width="16"/>
+</svg>

cmd/mbf/internal/pkgserver/ui/jstest/failure-open.svg

@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+ ! This view box is a bit weird: the strokes assume they're working in a view
+ ! box that spans from the (0,0) to (100,100), and indeed that is convenient
+ ! conceptualizing the strokes, but the stroke itself has a considerable width
+ ! that gets clipped by restrictive view box dimensions. Hence, the view is
+ ! shifted from (0,0)–(100,100) to (-20,-20)–(120,120), to make room for the
+ ! clipped stroke, while leaving behind an illusion of working in a view box
+ ! spanning from (0,0) to (100,100).
+ !
+ ! However, the resulting SVG is too close to the summary text, and CSS
+ ! properties to add padding do not seem to work with `content:` (likely because
+ ! they're anonymous replaced elements); thus, the width of the view is
+ ! increased considerably to provide padding in the SVG itself, while leaving
+ ! the strokes oblivious.
+ !
+ ! It gets worse: the summary text isn't vertically aligned with the icon! As
+ ! a flexbox cannot be used in a summary to align the marker with the text, the
+ ! simplest and most effective solution is to reduce the height of the view box
+ ! from 140 to 130, thereby removing some of the bottom padding present.
+ !
+ ! All six SVGs use the same view box (and indeed, they refer to this comment)
+ ! so that they all appear to be the same size and position relative to each
+ ! other on the DOM—indeed, the view box dimensions, alongside the width,
+ ! directly control their placement on the DOM.
+ !
+ ! TL;DR: CSS is janky, overflow is weird, and SVG is awesome!
+ -->
+<svg xmlns="http://www.w3.org/2000/svg" width="20" viewBox="-20,-20 160 130">
+    <!-- This triangle should match success-open.svg, fill and stroke color notwithstanding. -->
+    <polygon points="0,0 100,0 50,100" fill="red" stroke="red" stroke-width="15" stroke-linejoin="round"/>
+    <!-- See the comment in failure-closed.svg before modifying this. -->
+    <line x1="30" y1="10" x2="70" y2="50" stroke="white" stroke-width="16"/>
+    <line x1="30" y1="50" x2="70" y2="10" stroke="white" stroke-width="16"/>
+</svg>

cmd/mbf/internal/pkgserver/ui/jstest/index.html

@@ -18,9 +18,14 @@
 
 <main>
 <p id="counters">
-    <span id="successes">0</span> succeeded, <span id="failures">0</span> failed.
+    <span id="success-counter">0</span> succeeded, <span id="failure-counter">0</span>
+    failed<span id="skip-counter-text" hidden>, <span id="skip-counter">0</span> skipped</span>.
 </p>
 
+<p hidden id="success-description">Successful test</p>
+<p hidden id="failure-description">Failed test</p>
+<p hidden id="skip-description">Partially or fully skipped test</p>
+
 <div id="root">
 </div>
 

cmd/mbf/internal/pkgserver/ui/jstest/jstest.ts

@@ -63,28 +63,34 @@ function checkDuplicates(parent: string, names: { name: string }[]) {
     }
 }
 
-class FailNowSentinel {}
+export type TestState = "success" | "failure" | "skip";
+
+class AbortSentinel {}
 
 export class TestController {
+    #state: TestState;
     logs: string[];
-    #failed: boolean;
 
     constructor() {
+        this.#state = "success";
         this.logs = [];
-        this.#failed = false;
+    }
+
+    getState(): TestState {
+        return this.#state;
     }
 
     fail() {
-        this.#failed = true;
+        this.#state = "failure";
     }
 
     failed(): boolean {
-        return this.#failed;
+        return this.#state === "failure";
     }
 
     failNow(): never {
         this.fail();
-        throw new FailNowSentinel();
+        throw new AbortSentinel();
     }
 
     log(message: string) {
@@ -100,13 +106,23 @@ export class TestController {
         this.log(message);
         this.failNow();
     }
+
+    skip(message?: string): never {
+        if (message != null) this.log(message);
+        if (this.#state !== "failure") this.#state = "skip";
+        throw new AbortSentinel();
+    }
+
+    skipped(): boolean {
+        return this.#state === "skip";
+    }
 }
 
 // =============================================================================
 // Execution
 
 export interface TestResult {
-    success: boolean;
+    state: TestState;
     logs: string[];
 }
 
@@ -120,11 +136,11 @@ function runTests(reporter: Reporter, parents: string[], node: TestTree) {
     try {
         node.test(controller);
     } catch (e) {
-        if (!(e instanceof FailNowSentinel)) {
+        if (!(e instanceof AbortSentinel)) {
             controller.error(extractExceptionString(e));
         }
     }
-    reporter.update(path, { success: !controller.failed(), logs: controller.logs });
+    reporter.update(path, { state: controller.getState(), logs: controller.logs });
 }
 
 function extractExceptionString(e: any): string {
@@ -207,18 +223,20 @@ const SEP = " ❯ ";
 export class StreamReporter implements Reporter {
     stream: Stream;
     verbose: boolean;
+    #successes: ({ path: string[] } & TestResult)[];
     #failures: ({ path: string[] } & TestResult)[];
-    counts: { successes: number, failures: number };
+    #skips: ({ path: string[] } & TestResult)[];
 
     constructor(stream: Stream, verbose: boolean = false) {
         this.stream = stream;
         this.verbose = verbose;
+        this.#successes = [];
         this.#failures = [];
-        this.counts = { successes: 0, failures: 0 };
+        this.#skips = [];
     }
 
     succeeded(): boolean {
-        return this.counts.successes > 0 && this.counts.failures === 0;
+        return this.#successes.length > 0 && this.#failures.length === 0;
     }
 
     // We don't need the structure for reporting.
@@ -227,8 +245,9 @@ export class StreamReporter implements Reporter {
     update(path: string[], result: TestResult) {
         if (path.length === 0) throw new RangeError("path is empty");
         const pathStr = path.join(SEP);
-        if (result.success) {
-            this.counts.successes++;
+        switch (result.state) {
+        case "success":
+            this.#successes.push({ path, ...result });
             // NOTE: emojis are used instead of colored Unicode symbols as
             // coloring isn't possible through all streams, which would make
             // this terminal-specific, and even in terminals and emulators
@@ -241,31 +260,51 @@ export class StreamReporter implements Reporter {
             // also means you're more likely to have an emoji font but not
             // a font with those symbols—and that they're double-width.
             if (this.verbose) this.stream.writeln(`✅️ ${pathStr}`);
-        } else {
-            this.counts.failures++;
-            this.stream.writeln(`⚠️ ${pathStr}`);
+            break;
+        case "failure":
             this.#failures.push({ path, ...result });
+            this.stream.writeln(`⚠️ ${pathStr}`);
+            break;
+        case "skip":
+            this.#skips.push({ path, ...result });
+            this.stream.writeln(`⏭️ ${pathStr}`);
+            break;
         }
     }
 
     finalize() {
+        if (this.verbose) this.#displaySection("successes", this.#successes, true);
+        this.#displaySection("failures", this.#failures);
+        this.#displaySection("skips", this.#skips);
+        this.stream.writeln("");
+        this.stream.writeln(
+            `${this.#successes.length} succeeded, ${this.#failures.length} failed` +
+                (this.#skips.length ? `, ${this.#skips.length} skipped` : ""),
+        );
+    }
+
+    #displaySection(name: string, data: ({ path: string[] } & TestResult)[], ignoreEmpty: boolean = false) {
+        if (!data.length) return;
+
         // Transform [{ path: ["a", "b", "c"] }, { path: ["a", "b", "d"] }] into
         // { "a ❯ b": ["c", "d"] }. NOTE: intermediate nodes are collapsed as
         // excessive nesting is difficult to convey clearly in a text-only
         // environment.
         let pathMap = new Map<string, ({ name: string } & TestResult)[]>();
-        for (const f of this.#failures) {
-            if (f.path.length === 0) throw new RangeError("path is empty");
-            const key = f.path.slice(0, -1).join(SEP);
+        for (const t of data) {
+            if (t.path.length === 0) throw new RangeError("path is empty");
+            const key = t.path.slice(0, -1).join(SEP);
             if (!pathMap.has(key)) pathMap.set(key, []);
-            pathMap.get(key)!.push({ name: f.path.at(-1)!, ...f });
+            pathMap.get(key)!.push({ name: t.path.at(-1)!, ...t });
         }
 
         this.stream.writeln("");
-        this.stream.writeln("FAILURES");
-        this.stream.writeln("========");
+        this.stream.writeln(name.toUpperCase());
+        this.stream.writeln("=".repeat(name.length));
 
-        for (const [path, tests] of pathMap) {
+        for (let [path, tests] of pathMap) {
+            if (ignoreEmpty) tests = tests.filter((t) => t.logs.length);
+            if (tests.length === 0) continue;
             if (tests.length === 1) {
                 this.#writeOutput(tests[0], path ? `${path}${SEP}` : "", false);
             } else {
@@ -273,10 +312,6 @@ export class StreamReporter implements Reporter {
                 for (const t of tests) this.#writeOutput(t, "  - ", true);
             }
         }
-
-        this.stream.writeln("");
-        const { successes, failures } = this.counts;
-        this.stream.writeln(`${successes} succeeded, ${failures} failed`);
     }
 
     #writeOutput(test: { name: string } & TestResult, prefix: string, nested: boolean) {
@@ -315,11 +350,16 @@ export class DOMReporter implements Reporter {
 
     update(path: string[], result: TestResult) {
         if (path.length === 0) throw new RangeError("path is empty");
-        const counter = assertGetElementById(result.success ? "successes" : "failures");
+        if (result.state === "skip") {
+            assertGetElementById("skip-counter-text").hidden = false;
+        }
+        const counter = assertGetElementById(`${result.state}-counter`);
         counter.innerText = (Number(counter.innerText) + 1).toString();
+
         let parent = assertGetElementById("root");
         for (const node of path) {
             let child: HTMLDetailsElement | null = null;
+            let summary: HTMLElement | null = null;
             let d: Element;
             outer: for (d of parent.children) {
                 if (!(d instanceof HTMLDetailsElement)) continue;
@@ -327,6 +367,7 @@ export class DOMReporter implements Reporter {
                     if (!(s instanceof HTMLElement)) continue;
                     if (!(s.tagName === "SUMMARY" && s.innerText === node)) continue;
                     child = d;
+                    summary = s;
                     break outer;
                 }
             }
@@ -334,19 +375,41 @@ export class DOMReporter implements Reporter {
                 child = document.createElement("details");
                 child.className = "test-node";
                 child.ariaRoleDescription = "test";
-                const summary = document.createElement("summary");
+                summary = document.createElement("summary");
                 summary.appendChild(document.createTextNode(node));
                 summary.ariaRoleDescription = "test name";
                 child.appendChild(summary);
                 parent.appendChild(child);
             }
-            if (!result.success) {
-                // Minimize successes by default, by only expanding failures.
+            if (!summary) throw new Error("unreachable as assigned above");
+
+            switch (result.state) {
+            case "failure":
+                // Only expand failures, to minimize successes and skips.
                 child.open = true;
                 child.classList.add("failure");
+                child.classList.remove("skip");
+                child.classList.remove("success");
+                // The summary marker does not appear in the AOM, so setting its
+                // alt text is fruitless; label the summary itself instead.
+                summary.setAttribute("aria-labelledby", "failure-description");
+                break;
+            case "skip":
+                if (child.classList.contains("failure")) break;
+                child.classList.add("skip");
+                child.classList.remove("success");
+                summary.setAttribute("aria-labelledby", "skip-description");
+                break;
+            case "success":
+                if (child.classList.contains("failure") || child.classList.contains("skip")) break;
+                child.classList.add("success");
+                summary.setAttribute("aria-labelledby", "success-description");
+                break;
             }
+
             parent = child;
         }
+
         const p = document.createElement("p");
         p.classList.add("test-desc");
         if (result.logs.length) {
@@ -384,7 +447,13 @@ export class GoTestReporter implements Reporter {
     }
 
     update(path: string[], result: TestResult) {
-        console.log(JSON.stringify({ path, ...result }));
+        let state: number;
+        switch (result.state) {
+            case "success": state = 0; break;
+            case "failure": state = 1; break;
+            case "skip": state = 2; break;
+        }
+        console.log(JSON.stringify({ path, state, logs: result.logs }));
     }
 
     // Unnecessary but convenient on the Go side, so that it doesn't have to

cmd/mbf/internal/pkgserver/ui/jstest/skip-closed.svg

@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<!-- See failure-open.svg for an explanation of the view box dimensions. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="20" viewBox="-20,-20 160 130">
+    <!-- This triangle should match success-closed.svg, fill and stroke color notwithstanding. -->
+    <polygon points="0,0 100,50 0,100" fill="blue" stroke="blue" stroke-width="15" stroke-linejoin="round"/>
+    <!--
+     ! This path is extremely similar to the one in skip-open.svg; before
+     ! making minor modifications, diff the two to understand how they should
+     ! remain in sync.
+     -->
+    <path
+        d="M 50,50
+           A 23,23 270,1,1 30,30
+           l -10,20
+           m 10,-20
+           l -20,-10"
+        fill="none"
+        stroke="white"
+        stroke-width="12"
+        stroke-linejoin="round"/>
+</svg>

cmd/mbf/internal/pkgserver/ui/jstest/skip-open.svg

@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<!-- See failure-open.svg for an explanation of the view box dimensions. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="20" viewBox="-20,-20 160 130">
+    <!-- This triangle should match success-open.svg, fill and stroke color notwithstanding. -->
+    <polygon points="0,0 100,0 50,100" fill="blue" stroke="blue" stroke-width="15" stroke-linejoin="round"/>
+    <!--
+     ! This path is extremely similar to the one in skip-closed.svg; before
+     ! making minor modifications, diff the two to understand how they should
+     ! remain in sync.
+     -->
+    <path
+        d="M 50,50
+           A 23,23 270,1,1 70,30
+           l 10,-20
+           m -10,20
+           l -20,-10"
+        fill="none"
+        stroke="white"
+        stroke-width="12"
+        stroke-linejoin="round"/>
+</svg>

cmd/mbf/internal/pkgserver/ui/jstest/style.css

@@ -1,3 +1,8 @@
+/*
+ * When updating the theme colors, also update them in success-closed.svg and
+ * success-open.svg!
+ */
+
 :root {
   --bg: #d3d3d3;
   --fg: black;
@@ -34,8 +39,38 @@ details.test-node {
   > summary {
     cursor: pointer;
   }
+  &.success > summary::marker {
+    /*
+     * WebKit only supports color and font-size properties in ::marker [1], and
+     * its ::-webkit-details-marker only supports hiding the marker entirely
+     * [2], contrary to mdn's example [3]; thus, set a color as a fallback:
+     * while it may not be accessible for colorblind individuals, it's better
+     * than no indication of a test's state for anyone, as that there's no other
+     * way to include an indication in the marker on WebKit.
+     *
+     * [1]: https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Selectors/::marker#browser_compatibility
+     * [2]: https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/summary#default_style
+     * [3]: https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/summary#changing_the_summarys_icon
+     */
+    color: var(--fg);
+    content: url("./success-closed.svg") / "success";
+  }
+  &.success[open] > summary::marker {
+    content: url("./success-open.svg") / "success";
+  }
   &.failure > summary::marker {
     color: red;
+    content: url("./failure-closed.svg") / "failure";
+  }
+  &.failure[open] > summary::marker {
+    content: url("./failure-open.svg") / "failure";
+  }
+  &.skip > summary::marker {
+    color: blue;
+    content: url("./skip-closed.svg") / "skip";
+  }
+  &.skip[open] > summary::marker {
+    content: url("./skip-open.svg") / "skip";
   }
 }
 

cmd/mbf/internal/pkgserver/ui/jstest/success-closed.svg

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<!-- See failure-open.svg for an explanation of the view box dimensions. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="20" viewBox="-20,-20 160 130">
+    <style>
+    .adaptive-stroke {
+        stroke: black;
+    }
+    @media (prefers-color-scheme: dark) {
+        .adaptive-stroke {
+            stroke: ghostwhite;
+        }
+    }
+    </style>
+    <!-- When updating this triangle, also update the other five SVGs. -->
+    <polygon points="0,0 100,50 0,100" fill="none" class="adaptive-stroke" stroke-width="15" stroke-linejoin="round"/>
+</svg>

cmd/mbf/internal/pkgserver/ui/jstest/success-open.svg

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<!-- See failure-open.svg for an explanation of the view box dimensions. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="20" viewBox="-20,-20 160 130">
+    <style>
+    .adaptive-stroke {
+        stroke: black;
+    }
+    @media (prefers-color-scheme: dark) {
+        .adaptive-stroke {
+            stroke: ghostwhite;
+        }
+    }
+    </style>
+    <!-- When updating this triangle, also update the other five SVGs. -->
+    <polygon points="0,0 100,0 50,100" fill="none" class="adaptive-stroke" stroke-width="15" stroke-linejoin="round"/>
+</svg>

cmd/mbf/internal/pkgserver/ui/sample_test.ts

@@ -1,81 +0,0 @@
-import { NoOpReporter, TestRegistrar, context, group, suite, test } from "./jstest/jstest.js";
-
-suite("dog", [
-    group("tail", [
-        test("wags when happy", (t) => {
-            if (0 / 0 !== Infinity / Infinity) {
-                t.fatal("undefined must not be defined");
-            }
-        }),
-        test("idle when down", (t) => {
-            t.log("test test");
-            t.error("dog whining noises go here");
-        }),
-    ]),
-    test("likes headpats", (t) => {
-        if (2 !== 2) {
-            t.error("IEEE 754 violated: 2 is NaN");
-        }
-    }),
-    context("near cat", [
-        test("is ecstatic", (t) => {
-            if (("b" + "a" + + "a" + "a").toLowerCase() === "banana") {
-                t.error("🍌🍌🍌");
-                t.error("🍌🍌🍌");
-                t.error("🍌🍌🍌");
-                t.failNow();
-            }
-        }),
-        test("playfully bites cats' tails", (t) => {
-            t.log("arf!");
-            throw new Error("nom");
-        }),
-    ]),
-]);
-
-suite("cat", [
-    test("likes headpats", (t) => {
-        t.log("meow");
-    }),
-    test("tester tester", (t) => {
-        const r = new TestRegistrar();
-        r.suite("explod", [
-            test("with yarn", (t) => {
-                t.log("YAY");
-            }),
-        ]);
-        const reporter = new NoOpReporter();
-        r.run(reporter);
-        if (reporter.suites.length !== 1) {
-            t.fatal(`incorrect number of suites registered got=${reporter.suites.length} want=1`);
-        }
-        const suite = reporter.suites[0];
-        if (suite.name !== "explod") {
-            t.error(`suite name incorrect got='${suite.name}' want='explod'`);
-        }
-        if (suite.children.length !== 1) {
-            t.fatal(`incorrect number of suite children got=${suite.children.length} want=1`);
-        }
-        const test_ = suite.children[0];
-        if (test_.name !== "with yarn") {
-            t.error(`incorrect test name got='${test_.name}' want='with yarn'`);
-        }
-        if ("children" in test_) {
-            t.error(`expected leaf node, got group of ${test_.children.length} children`);
-        }
-        if (!reporter.finalized) t.error(`expected reporter to have been finalized`);
-        if (reporter.results.length !== 1) {
-            t.fatal(`incorrect result count got=${reporter.results.length} want=1`);
-        }
-        const result = reporter.results[0];
-        if (!(result.path.length === 2 &&
-              result.path[0] === "explod" &&
-              result.path[1] === "with yarn")) {
-            t.error(`incorrect result path got=${result.path} want=["explod", "with yarn"]`);
-        }
-        if (!result.success) t.error(`expected test to succeed`);
-        if (!(result.logs.length === 1 && result.logs[0] === "YAY")) {
-            t.error(`incorrect result logs got=${result.logs} want=["YAY"]`);
-        }
-    }),
-]);

cmd/mbf/internal/pkgserver/ui/test_ui.go

@@ -7,6 +7,7 @@ import "embed"
 //go:generate tsc -p tsconfig.test.json
 //go:generate cp index.html style.css favicon.ico static
 //go:generate cp jstest/index.html jstest/style.css static/jstest
+//go:generate sh -c "cp jstest/*.svg static/jstest"
 //go:embed static
 var _static embed.FS
 var static = staticFS(_static)