1
0
forked from rosa/hakurei

8 Commits

Author SHA1 Message Date
kat
90df5c13fb TODO: docs 2026-05-17 20:03:09 +10:00
kat
3b5b903f0e TODO: consider writing tests for the test runner. 2026-05-17 20:03:09 +10:00
kat
443db2d03d TODO: actually write tests lol. 2026-05-17 20:03:09 +10:00
kat
f2d883ee8c TODO: auto-load test files based on name, just like go (see long desc)
squash this into the commit that first added all_tests.ts, we don't even
want to have a trace of it left

for the cli ones, we can simply iterate the filesystem relative our
location. for the web one, we determine it on launch and expose it as an
endpoint from the server which the client queries
2026-05-17 20:03:09 +10:00
kat
536e3c97ea TODO: limited selective execution from cli (see long desc)
well the problem with arbitrary selection is that... you need to do lots
of matching, which is confusing too when you need to encode nesting. so
what if just.. node cli.js index_test.js?

this isn't concerned with reporters or execution, this happens at the
cli level and it solely affects which modules are imported instead of
just all_tests.js.

alternatively, we could do suites instead of files. this is probably
better huh because you don't need to type out all those file paths, and
it doesn't punish large files (because a test file corresponds to
a source code file)

so we'd just import all_tests.js, then just filter out suites whose name
doesn't match <input>, before calling `run` on it. deleting and
filtering out suites should probably be methods on the registrar

i suspect the impl will be tiny excl argument parser nonsense, so imo
squash this into the commit that added registrars

add a comment describing the use-case as “just run the tests i'm editing
to save time”, rather than as skipping, then briefly mention why general
purpose skipping is still a tentative future feature
2026-05-17 20:03:09 +10:00
kat
6a59d7df86 TODO: display elapsed time (see long description)
both on a test level and for the whole thing. i think the reporter or
registrar abstractions should deal with all timeouts, and just feed
elapsed time through all the functions: update() gets time for the
specific test, and finalize passes you the total time. this way you
don't need to do the same logic in every reporter, and you also give
a suggestion to reporter writers (i.e.: you in the future) to expose
test durations. actually tbh per-test isn't possible anywhere but in the
executor, especially when taking potential future parallel execution
into account

on the topic of parallelism: per-test is wall clock for that test,
regardless of perceived time, because no other number is useful. whole
thing is wall clock too, not cpu time

remember:
  - use monotonic clocks!! we need elapsed time, not absolute time
  - format them to more readable strings like “15h 12m” instead of
    “54738 seconds”. once things get large we can be less precise

for the go reporter: ask ozy if the go one already measures it. if so
then don't even bother serializing it

for the stream reporter: the live feed should include per-test time in
brackets or something. the final tree should only include timeout for
outliers on the long side (just shove a box plot-esque algo on it), and
if a flag is given print it for all nodes, and if another flag is given
print the n longest tests. the total time should be in the summary line
at the end in brackets à la pytest

for the dom reporter, we do the same as with the stream reporter's
outlier detection, and have a checkbox or button to dynamically
show/hide all timeouts, and another button to toggle a widget of sorts
that shows up right above the result tree which includes the n longest
tests. all these buttons should be on the same line as the summary
(successes/failures/skips). the total time should be included in the
“execution finished” text form the previous commit, i.e. “execution
finished in 15s”
2026-05-17 20:03:09 +10:00
kat
e198a98daa TODO: display text execution progress (see long description)
since the test tree is statically known, we also statically know how
many tests are present. we should hence be using this to provide
a counter, say [1/48], to give a rough estimate as to when tests might
finish. not a time estimate of course, since we can't determine that

nota bene, we can't pass the current test count, and instead need to let
the reporter deal with that, since otherwise we can't easily parallelize
execution in the future. definitely mention this in a comment somewhere
to elaborate on the design

for the go reporter, ask ozy if go has any way to tell it this info.
i doubt it since they don't have a statically known test count. if it
does, then just send the count alongside the tree

for the stream reporter, ignore it entirely; we don't even display
successes by default so the number has nowhere to be attached to

for the dom reporter, put it somewhere in the header, i think alongside
the success/failure/skip count. something like “in progress (4/28)”.
then once finalize() is called change the whole thing to “execution
finished”
2026-05-17 20:03:09 +10:00
kat
4f0841e1eb WIP: cmd/mbf: jstest: add JSON reporter for go test integration 2026-05-17 20:03:09 +10:00
5 changed files with 63 additions and 69 deletions

View File

@@ -1,2 +1,2 @@
// Import all test files to register their test suites.
import "./sample_test.js";
import "./index_test.js";

View File

@@ -0,0 +1,2 @@
import { suite, test } from "./jstest/jstest.js";
import "./index.js";

View File

@@ -0,0 +1,3 @@
import "../all_tests.js";
import { GoTestReporter, GLOBAL_REGISTRAR } from "./jstest.js";
GLOBAL_REGISTRAR.run(new GoTestReporter());

View File

@@ -23,6 +23,7 @@ export class TestRegistrar {
}
run(reporter: Reporter) {
reporter.register(this.#suites);
for (const suite of this.#suites) {
for (const c of suite.children) runTests(reporter, [suite.name], c);
}
@@ -157,6 +158,9 @@ function extractExceptionString(e: any): string {
// Reporting
export interface Reporter {
// A notable feature—or flaw, to some—of the DSL is that the tree of tests
// is statically known, which might greatly aid in implementing a reporter.
register(suites: TestGroup[]): void;
// While we could simply call a function with a tree representing all
// results, which would indeed greatly simplify implementation of reporters,
// simply registering a path and allowing the reporter to—either implicitly
@@ -186,14 +190,20 @@ export interface Reporter {
// underlying result tree; this makes it extremely convenient in some cases like
// testing ourself.
export class NoOpReporter implements Reporter {
suites: TestGroup[];
results: ({ path: string[] } & TestResult)[];
finalized: boolean;
constructor() {
this.suites = [];
this.results = [];
this.finalized = false;
}
register(suites: TestGroup[]) {
this.suites = suites;
}
update(path: string[], result: TestResult) {
this.results.push({ path, ...result });
}
@@ -229,6 +239,9 @@ export class StreamReporter implements Reporter {
return this.#successes.length > 0 && this.#failures.length === 0;
}
// We don't need the structure for reporting.
register(suites: TestGroup[]) {}
update(path: string[], result: TestResult) {
if (path.length === 0) throw new RangeError("path is empty");
const pathStr = path.join(SEP);
@@ -342,6 +355,13 @@ function assertGetElementById(id: string): HTMLElement {
// A reporter that directly translates a tree of results into a tree of
// collapsible elements in the DOM.
export class DOMReporter implements Reporter {
// It is very difficult to implement this using the statically known tree,
// because Map doesn't handle array keys properly (to store the path), and
// it's unknown of there's any way to implement it without writing one's own
// data types. Oh well; using the DOM as a data structure might seem hacky
// but it does have its benefits, apart from encouraging a tagless final.
register(suites: TestGroup[]) {}
update(path: string[], result: TestResult) {
if (path.length === 0) throw new RangeError("path is empty");
if (result.state === "skip") {
@@ -419,3 +439,40 @@ export class DOMReporter implements Reporter {
finalize() {}
}
interface GoNode {
name: string;
subtests?: GoNode[];
}
// Used to display results via `go test`, via some glue code from the Go side.
// TODO(ophestra): said glue code has to be written.
export class GoTestReporter implements Reporter {
register(suites: TestGroup[]) {
console.log(JSON.stringify(suites.map(GoTestReporter.serialize)));
}
// Convert a test tree into the one expected by the Go code.
static serialize(node: TestTree): GoNode {
return {
name: node.name,
subtests: "children" in node ? node.children.map(GoTestReporter.serialize) : undefined,
};
}
update(path: string[], result: TestResult) {
let state: number;
switch (result.state) {
case "success": state = 0; break;
case "failure": state = 1; break;
case "skip": state = 2; break;
}
console.log(JSON.stringify({ path, state, logs: result.logs }));
}
// Unnecessary but convenient on the Go side, so that it doesn't have to
// infer this via process exit.
finalize() {
console.log("null");
}
}

View File

@@ -1,68 +0,0 @@
import { NoOpReporter, TestRegistrar, context, group, suite, test } from "./jstest/jstest.js";
suite("dog", [
group("tail", [
test("wags when happy", (t) => {
if (0 / 0 !== Infinity / Infinity) {
t.fatal("undefined must not be defined");
}
}),
test("idle when down", (t) => {
t.log("test test");
t.error("dog whining noises go here");
}),
]),
test("likes headpats", (t) => {
if (2 !== 2) {
t.error("IEEE 754 violated: 2 is NaN");
}
}),
context("near cat", [
test("is ecstatic", (t) => {
if (("b" + "a" + + "a" + "a").toLowerCase() === "banana") {
t.error("🍌🍌🍌");
t.error("🍌🍌🍌");
t.error("🍌🍌🍌");
t.failNow();
}
}),
test("playfully bites cats' tails", (t) => {
t.log("arf!");
throw new Error("nom");
}),
]),
]);
suite("cat", [
test("likes headpats", (t) => {
t.log("meow");
}),
test("owns skipping rope", (t) => {
t.skip("this cat is stuck in your machine!");
t.log("never logged");
}),
test("tester tester", (t) => {
const r = new TestRegistrar();
r.suite("explod", [
test("with yarn", (t) => {
t.log("YAY");
}),
]);
const reporter = new NoOpReporter();
r.run(reporter);
if (!reporter.finalized) t.error(`expected reporter to have been finalized`);
if (reporter.results.length !== 1) {
t.fatal(`incorrect result count got=${reporter.results.length} want=1`);
}
const result = reporter.results[0];
if (!(result.path.length === 2 &&
result.path[0] === "explod" &&
result.path[1] === "with yarn")) {
t.error(`incorrect result path got=${result.path} want=["explod", "with yarn"]`);
}
if (result.state !== "success") t.error(`expected test to succeed`);
if (!(result.logs.length === 1 && result.logs[0] === "YAY")) {
t.error(`incorrect result logs got=${result.logs} want=["YAY"]`);
}
}),
]);