{ "schema": "router-school.ledger.v1", "generated_from": "gpu-compute-harness/references (Owlforge Router School, 2026-08-04)", "last_updated": "2026-08-04", "program": { "id": "router-school", "title": "Router School", "tagline": "A closed research program that tried to learn the dispatch policy that routes work to the right provider — and kept failing it on purpose until something survived. Final research date 2026-08-04.", "origin_campaign": "owlforge.overnight.2026-08-04", "node": "RTX 3090 Ti · local node", "teacher": "dispatch_guard.py::assess_dispatch", "license": "CC-BY-4.0 — research artifacts" }, "current_conclusion": { "as_of": "2026-08-04", "production_artifact": "deterministic rule", "headline": "The recovered deterministic teacher rule scores 100% on the V5 test set. It stays in production. The V5 tiny MLP is retained as a shadow benchmark, not a promoted model.", "promoted_model": false, "promotion_statement": "No learned model has been promoted to the live routing path at any point in this program.", "stats": [ { "label": "V5 accuracy", "value": "99.05%", "sub": "104 / 105 test rows" }, { "label": "V5 macro-F1", "value": "0.9831", "sub": "weighted-F1 0.9905" }, { "label": "Strict validity", "value": "100%", "sub": "no JSON generation step" }, { "label": "Stdlib CPU p95", "value": "0.0773 ms", "sub": "budget 250 ms" } ], "counterweights": [ "The recovered teacher rule scores 100% on the same 105-example test set — strictly above the model's 99.05%.", "The rule is ~20 lines of stdlib, impossible to misconfigure, and needs no checkpoint.", "Out of distribution the model drops to 93.40% accuracy / 0.9260 macro-F1 on a 500-cell window of never-seen counter values.", "A learned classifier that matches a deterministic teacher is evidence the architectural ladder was mis-chosen, not a shipping candidate." ] }, "ladder": { "title": "Classification ladder", "rungs": [ { "n": 1, "name": "Rule + lookup", "note": "the task-class rule baseline" }, { "n": 2, "name": "Sparse linear", "note": "TF-IDF + logistic regression" }, { "n": 3, "name": "Tiny MLP", "note": "1–2 hidden layers, ~50–1,000 parameters" }, { "n": 4, "name": "Boosted trees", "note": "often the best accuracy/latency tradeoff for tabular" }, { "n": 5, "name": "Generative decoder", "note": "last resort — only when the task is genuinely generative" } ], "rule": "Classification tasks start at rung 3 and climb left only if the MLP fails. Generative tasks start at rung 5 (rank-16 QLoRA) and climb down." }, "cells": [ { "id": "v1", "label": "V1", "title": "Round-1 shadow eval", "subtitle": "QLoRA adapter vs a frozen deterministic baseline", "date": "2026-08-04", "window": "06:23–06:41 UTC (eval) · 12:29 UTC (audit)", "verdict": "refuted", "verdict_label": "FALSIFIED", "status_note": "Historical falsification. Not the current status of the program.", "historical": true, "headline": { "label": "Claimed delta_lane_accuracy", "value": "+5.10 pp", "delta": "adapter 53.06% vs baseline 47.96% · n=98 chronological holdout" }, "architecture": "rank-16 QLoRA adapter, generative decoder", "dataset": "chronological holdout, 98 test rows, no dedup", "why_it_advanced": "The headline delta looked promotion-grade: the eval runner exited 0, every receipt field was populated, the pipeline wrote the public site, and the Pages deploy succeeded.", "why_it_failed": "Two independent audit lanes (Grok 4.5, 337 lines; Kimi K3, 257 lines) recomputed every claim from the raw artifacts with read-only tools and converged on identical findings without seeing each other. The delta was a trivial-rule match sitting on top of leakage and majority-class collapse.", "metrics": [ { "label": "Adapter accuracy", "value": "53.06%", "note": "n=98" }, { "label": "Baseline accuracy", "value": "47.96%", "note": "hardcoded literal, not recomputed" }, { "label": "Adapter macro-F1", "value": "0.326", "note": "bit-identical to the 5-line rule" }, { "label": "Rule macro-F1", "value": "0.326", "note": "task_class → lane lookup" }, { "label": "Macro-F1 delta", "value": "0.0007", "note": "rule parity" }, { "label": "Bootstrap CI95 (accuracy delta)", "value": "[−8.2, +18.4] pp", "note": "spans zero" }, { "label": "McNemar exact two-sided", "value": "p ≈ 0.55", "note": "b=20, c=25 (p=0.5515)" }, { "label": "Honest delta on deduped rows", "value": "≈ +2.6 pp", "note": "76 clean rows after removing collisions" }, { "label": "Train/test byte collisions", "value": "46 / 98", "note": "22 of them verbatim training strings" } ], "gates": [ { "gate": "Trivial-rule parity", "bar": "adapter macro-F1 > rule macro-F1 by a margin surviving the bootstrap CI", "actual": "adapter 0.326 / rule 0.326 — bit-identical", "pass": false }, { "gate": "Leakage-free holdout", "bar": "zero prompt bodies collide with train", "actual": "46/98 byte-collide; 22/98 verbatim train strings", "pass": false }, { "gate": "Macro-F1 over accuracy", "bar": "macro-F1 delta with CI excluding 0", "actual": "delta 0.0007; accuracy CI spans zero", "pass": false }, { "gate": "Bootstrap CI significance", "bar": "CI95 on per-example accuracy delta excludes 0", "actual": "[−8.2, +18.4] pp", "pass": false }, { "gate": "McNemar exact two-sided", "bar": "p < 0.05 on discordant pairs", "actual": "b=20, c=25 → p ≈ 0.55", "pass": false }, { "gate": "No majority-class collapse", "bar": "per-class recall ≥ ε for every class", "actual": "minimax predicted 72/98; claude recall 0.04; zai never predicted", "pass": false }, { "gate": "No hardcoded numbers", "bar": "baseline recomputed from real inference", "actual": "47.96% literal in the eval runner source", "pass": false } ], "gate_note": "The source reference records the verdict as 5/7 gates failed → falsified, while its per-check table marks all seven checks failing. Both statements are reproduced here rather than reconciled, because reconciling them would mean inventing a number.", "receipt_defects": [ "valid_json accepted any parseable dict — no schema check on the required keys.", "Raw generations were never stored; only the parsed object was persisted.", "The keyword-baseline fit() was a no-op, so the embedding baseline never ran.", "The 972-row experiment was not covered by the preregistration.", "The chronological split was mechanically correct but semantically broken by drift: after dedup and latest-wins, test flipped to 7% minimax from 72% in train." ], "correction": { "summary": "When promotion gates fail, the response is not a rerun with tweaked hyperparameters and not shipping with caveats — it is correcting the public record first.", "branch": "fix/router-school-audit-verdict", "head": "1083652a6e38e1f06c9515f81f8389abd161528b", "merge_commit": "e5b2dc4144aa49ebf5adc976f9b06dd6450aca08", "pipelines": "15881 (MR) and 15882 (main post-merge) — all jobs success with allow_failure=false", "read_back": "version 0.2.1 · station status failed · campaign 3 completed / 1 failed" }, "artifacts": [ { "name": "Public site record", "path": "data/projects/router-school.json", "visibility": "public", "sha256": null, "note": "no private prompt text" }, { "name": "Row-level eval output", "path": "tooling/artifacts/router-school-eval.json", "visibility": "private", "sha256": null, "note": "mode 0600 — body withheld" }, { "name": "Independent audit transcripts", "path": "Grok 4.5 (337 lines) · Kimi K3 (257 lines)", "visibility": "private", "sha256": null, "note": "dispatch-local, bodies withheld" } ] }, { "id": "v2", "label": "V2", "title": "Foundation rebuild and bounded GPU matrix", "subtitle": "Strict TDD dataset + evaluator, then three adapters on a leakage-free holdout", "date": "2026-08-04", "window": "post-audit", "verdict": "refuted", "verdict_label": "FALSIFIED", "status_note": "The foundation survived and is still in use. The three trained cells did not.", "historical": true, "headline": { "label": "Best cell macro-F1", "value": "0.1905", "delta": "vs task-class rule 0.2351 — the rule still wins" }, "architecture": "rank-16 and rank-32 QLoRA on unsloth/Llama-3.2-3B-Instruct", "dataset": "972 raw rows → 601 unique signatures → signature-grouped split 480 train / 60 val / 61 test", "why_it_advanced": "V1's lesson was not 'train one more epoch'. Every V1 defect was closed structurally before a single GPU cell was scheduled: dedup moved into the split itself, baselines recomputed from live inference, raw generations persisted, strict schema validation, and a preregistration written before training.", "why_it_failed": "All three GPU cells failed the promotion gate. None beat the modal task_class → lane rule (macro-F1 0.2351) by a margin whose CI excluded zero. The matrix was the right experiment and it correctly killed V2.", "foundation": { "tests": "65 → 87 green (+22 across seven RED-GREEN slices; no parent test weakened or skipped)", "fixes": [ { "defect": "972 raw rows with no dedup", "fix": "972 → 601 unique signatures" }, { "defect": "Conflicting duplicate rows kept verbatim", "fix": "26 conflicting signatures / 167 rows resolved latest-wins" }, { "defect": "Cross-split leakage possible", "fix": "0 cross-split signature collisions, asserted by test" }, { "defect": "Random split with no signature grouping", "fix": "signature-grouped split 480 / 60 / 61" }, { "defect": "Test label distribution not asserted", "fix": "independently matches the audit: claude 21 / grok 21 / zai 9 / kimi 6 / minimax 4" }, { "defect": "No determinism guarantee", "fix": "deterministic rebuild test plus sha256sum -c after every build" } ], "schema": "strict valid_json requires exactly four keys — task_class, recommended_lane, model, confidence — with enum values derived from the corpus, not hardcoded" }, "metrics": [ { "label": "Majority baseline", "value": "acc 0.0656 / macro-F1 0.0246", "note": "61 test rows" }, { "label": "Task-class rule baseline", "value": "acc 0.3115 / macro-F1 0.2351", "note": "the parity ceiling" }, { "label": "TF-IDF + logreg baseline", "value": "acc 0.3115 / macro-F1 0.2351", "note": "runtime 0.025 s; substituted for the unavailable embedding baseline and recorded in the preregistration before the harness shipped" }, { "label": "E2 — V1 adapter on V2 holdout", "value": "acc 18.03% / macro-F1 0.1297 / JSON 52.46%", "note": "worse than the rule on every class" }, { "label": "E3 — fresh rank-16 QLoRA (r=16, α=32)", "value": "acc 29.51% / macro-F1 0.1905 / JSON 96.72%", "note": "within noise of the rule" }, { "label": "E4 — class-balanced rank-32 (80× zai oversample)", "value": "acc 0.00% / macro-F1 0.0000 / JSON 18.03%", "note": "generation collapse" } ], "gates": [ { "gate": "Adapter macro-F1 > task-class rule", "bar": "by ≥ CI half-width", "actual": "best cell 0.1905 vs rule 0.2351", "pass": false }, { "gate": "Bootstrap CI95 on accuracy delta", "bar": "excludes 0", "actual": "no cell cleared it", "pass": false }, { "gate": "McNemar exact two-sided", "bar": "p < 0.05", "actual": "no cell cleared it", "pass": false }, { "gate": "All five providers have recall > 0", "bar": "yes", "actual": "E4 collapsed entirely", "pass": false }, { "gate": "Strict JSON validity rate", "bar": "≥ 95%", "actual": "E2 52.46% · E3 96.72% · E4 18.03%", "pass": false } ], "diagnostic": "E2 vs E3 is the load-bearing reading. A clean adapter on fresh data gained 11.5 pp of accuracy and 44 pp of JSON validity over the V1 adapter, but macro-F1 stayed within noise of the rule. Data cleaning fixed the JSON-generation failure and changed none of the routing decisions — the adapter was learning what the rule already captures.", "artifacts": [ { "name": "V2 baselines", "path": "tooling/artifacts/router-school-v2-baselines.json", "visibility": "public", "sha256": null, "note": "majority + task-class rule" }, { "name": "V2 TF-IDF logreg baseline", "path": "tooling/artifacts/router-school-v2-tfidf-logreg.json", "visibility": "public", "sha256": null, "note": "balanced word 1–2gram" }, { "name": "V2 fixtures + manifest", "path": "tooling/fixtures/v2/{train,val,test,manifest}.json", "visibility": "public", "sha256": null, "note": "deterministic rebuild verified by sha256sum -c" }, { "name": "Preregistration", "path": "tooling/ROUTER_SCHOOL_V2_PREREGISTRATION.md", "visibility": "public", "sha256": null, "note": "P1–P9 gates written before training" } ] }, { "id": "v3", "label": "V3", "title": "Current-policy distillation", "subtitle": "Train against the live teacher, with its hidden state made explicit", "date": "2026-08-04", "window": "13:42 UTC (fixtures)", "verdict": "partial", "verdict_label": "EVAL INVALIDATED", "status_note": "The first V3 evaluation measured the harness, not the adapter. The curriculum insight survived and became V4.", "historical": true, "headline": { "label": "First eval result", "value": "accuracy 0.0 / JSON validity 0.0", "delta": "traced to a duplicated response marker in the evaluator, not to the adapter" }, "architecture": "rank-16 QLoRA on unsloth/Llama-3.2-3B-Instruct — 700 steps, lr 1.5e-4, 1,048 examples", "dataset": "1,310 unique policy cells distilled from the live teacher · splits 1,048 train / 131 val / 131 test", "why_it_advanced": "V2 proved the dataset itself was the wrong signal. The production label is a function of (prompt, recent 24h provider counts) — the dispatch guard's portfolio-balancing sort. V2 dropped that history field, so identical prompts carried conflicting labels resolved by latest-wins, and the adapter learned 'the most recent conflicting label'. V3 makes the hidden variable explicit and trains against the deterministic teacher the live router actually runs, so there is no drift between training and serving.", "why_it_failed": "The evaluator built the generation prefix twice, appending a second response marker after the instruction template already ended with one. Every adapter scored through that harness emitted a scalar cascade instead of JSON. The fingerprint — accuracy 0 and json_validity 0 and every baseline delta ≤ 0 with probability 1.0 — is the signature of a prompt-template mismatch, not a bad adapter.", "dataset_shape": { "static_cells": "810 — 10 task classes × 3 steps × 3 lanes × 3 files × 3 minutes", "fairness_cells": "500 background cells sampled from a 1,024-state recent-count grid over {0, 3, 6, 9}, shuffled with a fixed seed", "total": "1,310 unique examples after dedup by text hash", "inputs": "task_class (10 enum values), must_delegate, lanes, estimated steps / files / minutes, and recent_ counts for claude, grok, kimi, minimax, zai", "schema": "owlforge.router-school.policy-distill.v3" }, "metrics": [ { "label": "Balanced TF-IDF on the V3 test split", "value": "macro-F1 0.7559", "note": "131 examples, 100% zai recall — the fairness-state features do carry learnable signal" }, { "label": "First adapter eval (invalidated)", "value": "accuracy 0.0 / macro-F1 0.0 / JSON validity 0.0", "note": "harness defect, not model quality" }, { "label": "Harness test suite after the fix", "value": "104 → 105 green", "note": "new test asserts exactly one response marker" } ], "gates": [ { "gate": "Evaluator measures the model, not itself", "bar": "exactly one response marker in the inference prompt", "actual": "two markers — every score through this harness was uninformative", "pass": false } ], "fix": "A single canonical inference_prompt() helper returning the exact prefix with one marker and a trailing space, landed RED-GREEN with a test that asserts the marker count. Three already-trained adapters were re-queued for re-evaluation without retraining — the adapters were never at fault, so no additional GPU hours were spent on identical weights.", "diagnostic": "The structural reading that drove V4: the recent-count signal is exercised on only a thin fraction of the 1,310-cell space. Most prompt-time cells pick by fit score and never touch the portfolio-balancing sort, so the adapter never specialized on the decision that actually matters.", "artifacts": [ { "name": "V3 train SFT manifest", "path": "tooling/fixtures/v3-policy-distill/", "visibility": "public", "sha256": "b6de0235529d23f47a42d0dcbc6bc368b373b68159e95a0c314738121086c864", "note": "manifest SHA-256 of the train SFT split" }, { "name": "V3 policy-distill eval (invalidated)", "path": "tooling/artifacts/router-school-v3-policy-r16-eval.json", "visibility": "public", "sha256": null, "note": "retained as the record of the harness defect" } ] }, { "id": "v4", "label": "V4", "title": "Fairness specialist (3B)", "subtitle": "First cell in the program to pass all five promotion gates", "date": "2026-08-04", "window": "14:08 UTC (dataset) · 14:21 UTC (cell)", "verdict": "partial", "verdict_label": "GATES PASSED · LATENCY FAILED", "status_note": "Quality gates cleared; the latency budget did not. Superseded by V5, not promoted.", "historical": true, "headline": { "label": "Macro-F1", "value": "0.6912", "delta": "CI95 0.5886–0.7734 · +0.5890 over majority, +0.6441 over TF-IDF" }, "architecture": "rank-16 QLoRA on unsloth/Llama-3.2-3B-Instruct", "dataset": "5⁵ = 3,125 recent-count states over {0, 3, 6, 9, 12}, sampled to 1,024 by deterministic stride, 818 retained after the confident-selection filter · 818 train / 105 test, signature-grouped", "why_it_advanced": "V3 showed the fairness surface was the load-bearing signal and that it was too thinly sampled to learn. V4 isolates it: a stratified grid over the five-provider recent-count state space with a fixed task_class and a fixed must_delegate flag, so the student learns the portfolio-balancing decision without the noise of fit-score dispatch on unrelated task classes. Stride sampling rather than random shuffle keeps edge states — one provider at 12, the rest at 0 — represented in the test split instead of clustering it around the modal tier.", "why_it_failed": "The registered p95 latency target is 250 ms. V4 measured 1,635.9 ms — 6.5× over budget. The fairness decision was correct and the adapter was too slow to run in the live router.", "metrics": [ { "label": "Accuracy", "value": "68.57%", "note": "105-example test set" }, { "label": "Macro-F1", "value": "0.6912", "note": "CI95 0.5886–0.7734" }, { "label": "Weighted-F1", "value": "0.6867", "note": "" }, { "label": "Strict JSON validity", "value": "100%", "note": "" }, { "label": "p95 latency", "value": "1,635.9 ms", "note": "budget 250 ms — 6.5× over" }, { "label": "Runtime", "value": "305.554 s", "note": "peak util 92% · peak VRAM 3,180 MiB · peak power 332.8 W · peak temp 68 °C" }, { "label": "Majority baseline", "value": "acc 0.3429 / macro-F1 0.1021", "note": "collapses to MiniMax; the task-class rule is identical because V4 fixes task_class" }, { "label": "TF-IDF + balanced logreg baseline", "value": "acc 0.1333 / macro-F1 0.0471", "note": "collapsed entirely to Claude — sparse word features cannot generalize a numeric ordering" } ], "per_class": { "columns": ["Provider", "Recall", "F1"], "rows": [ ["Claude", "92.86%", "0.8387"], ["Grok", "47.37%", "0.6429"], ["Kimi", "70.00%", "0.5833"], ["MiniMax", "66.67%", "0.6154"], ["Z.ai", "73.08%", "0.7755"] ] }, "gates": [ { "gate": "Adapter macro-F1 > task-class rule", "bar": "by ≥ CI half-width", "actual": "0.6912 vs 0.1021 (CI half-width 0.0924)", "pass": true }, { "gate": "Bootstrap CI95 on accuracy delta", "bar": "excludes 0", "actual": "[0.6, 0.7714]", "pass": true }, { "gate": "McNemar exact two-sided", "bar": "p < 0.05", "actual": "1.06e-16 vs TF-IDF · 3.18e-6 vs majority", "pass": true }, { "gate": "All five providers have recall > 0", "bar": "yes", "actual": "minimum 47.37% (Grok)", "pass": true }, { "gate": "Strict JSON validity rate", "bar": "≥ 95%", "actual": "100%", "pass": true }, { "gate": "p95 latency budget", "bar": "≤ 250 ms", "actual": "1,635.9 ms", "pass": false } ], "artifacts": [ { "name": "V4 dataset manifest", "path": "tooling/fixtures/v4-fairness-specialist/manifest.json", "visibility": "public", "sha256": "e082229bab312618f4d4e4c1590dcee3951b03d7734bc45d1c2af8d0e9dab8d7", "note": "schema owlforge.router-school.fairness-specialist.v4" }, { "name": "V4 train SFT split", "path": "tooling/fixtures/v4-fairness-specialist/train-sft.json", "visibility": "public", "sha256": "5abcd931c25d8a56ca73cc6cfba4f8f167f8557f8e0fe5436b2fb0e59d86201a", "note": "" }, { "name": "V4 train split", "path": "tooling/fixtures/v4-fairness-specialist/train.json", "visibility": "public", "sha256": "d10b425b4bbd706ecf3d8f722ebe4c60c82744987424de42342bb03587107cac", "note": "" }, { "name": "V4 test split", "path": "tooling/fixtures/v4-fairness-specialist/test.json", "visibility": "public", "sha256": "1e9b03dca19423b08ed1396b3b3ae3999813a2b3869edcc48946a1fb9534f037", "note": "105 rows — the frozen window every later cell is scored on" }, { "name": "V4 adapter", "path": "artifacts/checkpoints/router-v4-fairness-r16-lora", "visibility": "public", "sha256": "b44784f8f0283fcb2169b418de7d10dc186dc80277f663e6bd81c9d466e89837", "note": "adapter SHA pinned in both the train and eval wrappers" }, { "name": "V4 baselines", "path": "tooling/artifacts/router-school-v4-baselines.json", "visibility": "public", "sha256": null, "note": "written before the GPU cell shipped" } ] }, { "id": "v4-1b", "label": "V4-1B", "title": "1B distillation", "subtitle": "The 'smaller decoder' answer to the latency gate", "date": "2026-08-04", "window": "14:25 UTC", "verdict": "refuted", "verdict_label": "FALSIFIED", "status_note": "Failed every promotion gate. Retained as the empirical proof that the generative route was the wrong architecture.", "historical": true, "headline": { "label": "Accuracy", "value": "13.33%", "delta": "−55.24 pp against the 3B cell, for a 1.6× latency win" }, "architecture": "rank-16 QLoRA on unsloth/Llama-3.2-1B-Instruct", "dataset": "same frozen V4 train/test signatures", "why_it_advanced": "V4's quality was promotion-grade and only latency blocked it, so the registered next arm was the obvious one: hold the curriculum, shrink the base model, and check whether the fairness signal survives a 3× parameter reduction.", "why_it_failed": "It did not survive. The 1B adapter defaulted to emitting raw scalar tokens for almost every prompt; only 14 of 105 test rows produced parseable JSON. It never learned the fairness policy — it learned that emitting a scalar is the lowest-loss response.", "metrics": [ { "label": "Accuracy", "value": "13.33%", "note": "vs 68.57% at 3B — −55.24 pp" }, { "label": "Macro-F1", "value": "0.1292", "note": "vs 0.6912 at 3B — −0.5620" }, { "label": "Strict JSON validity", "value": "13.33%", "note": "vs 100% at 3B — −86.67 pp · 14 of 105 rows parseable" }, { "label": "p95 latency", "value": "954.08 ms", "note": "1.6× faster than 3B, still 3.8× over the 250 ms budget" }, { "label": "Runtime", "value": "110.253 s", "note": "2.77× faster · peak VRAM 1,790 MiB (43.7% lower) · peak power 315.81 W · peak temp 65 °C" }, { "label": "Adapter size", "value": "45,118,424 bytes", "note": "" }, { "label": "Cost per cell", "value": "1.8 min", "note": "vs 5.1 min at 3B — 2.8× cheaper and worthless" } ], "gates": [ { "gate": "Adapter macro-F1 > task-class rule", "bar": "by ≥ CI half-width", "actual": "0.1292", "pass": false }, { "gate": "Bootstrap CI95 on accuracy delta", "bar": "excludes 0", "actual": "not cleared", "pass": false }, { "gate": "McNemar exact two-sided", "bar": "p < 0.05", "actual": "not cleared", "pass": false }, { "gate": "All five providers have recall > 0", "bar": "yes", "actual": "kimi never predicted; 91 of 105 rows unparseable", "pass": false }, { "gate": "Strict JSON validity rate", "bar": "≥ 95%", "actual": "13.33%", "pass": false }, { "gate": "p95 latency budget", "bar": "≤ 250 ms", "actual": "954.08 ms", "pass": false } ], "diagnostic": "The V4 fairness task is not a generative task. It is five-class numeric classification over five input counters. The smallest viable generative decoder is still multiples over the latency budget because it has to autoregress an entire JSON document to produce one integer. A latency-budget violation on a learned adapter is a structural signal, not a parameter-tuning problem: the next arm is not 'smaller decoder', it is 're-derive the architecture from the task shape'.", "artifacts": [ { "name": "V4-1B adapter", "path": "artifacts/checkpoints/router-v4-fairness-1b-r16-lora", "visibility": "public", "sha256": "73fb4805a9ea32aa210bf60074f719317d8f8bfed13517d985ca7cb34e3887f3", "note": "" }, { "name": "V4-1B public eval", "path": "tooling/artifacts/router-school-v4-fairness-1b-r16-eval.json", "visibility": "public", "sha256": null, "note": "" }, { "name": "V4-1B private eval", "path": "tooling/artifacts/router-school-v4-fairness-1b-r16-eval-private.json", "visibility": "private", "sha256": null, "note": "raw generations — body withheld" } ] }, { "id": "v5", "label": "V5", "title": "Tiny numeric MLP", "subtitle": "The right rung of the classification ladder — and the cell that found the rule", "date": "2026-08-04", "window": "14:47 UTC (receipt)", "verdict": "confirmed", "verdict_label": "SHADOW BENCHMARK", "status_note": "Current cell. Every quality and latency gate cleared — and the recovered teacher rule still beats it, so the rule stays in production.", "historical": false, "headline": { "label": "Accuracy", "value": "99.05%", "delta": "104 / 105 · macro-F1 0.9831 · stdlib CPU p95 0.0773 ms" }, "architecture": "10×32×32×5 ReLU MLP trained with PyTorch — 1,573 parameters, no tokenizer, no autoregressive loop", "dataset": "the same frozen V4 fairness dataset and 105-row test split", "why_it_advanced": "V4-1B closed the generative ladder. For a five-class numeric classification over five counters the correct curriculum is rule → sparse linear → tiny MLP → boosted trees → generative decoder, and classification starts at the third rung. V5 picks that rung on the identical dataset, and the output is a class label rather than a generated document.", "why_it_failed": null, "metrics": [ { "label": "Accuracy", "value": "99.05%", "note": "104 / 105" }, { "label": "Macro-F1", "value": "0.9831", "note": "" }, { "label": "Weighted-F1", "value": "0.9905", "note": "" }, { "label": "Strict JSON validity", "value": "100%", "note": "no JSON is generated — the class is the output" }, { "label": "Bootstrap accuracy CI95", "value": "97.14–100%", "note": "" }, { "label": "Bootstrap macro-F1 CI95", "value": "0.9398–1.0", "note": "" }, { "label": "Stdlib CPU p95", "value": "0.0773 ms", "note": "the deployment target — ~12,000× faster than the failed 1B decoder and ~21,000× faster than the 3B QLoRA adapter" }, { "label": "Torch CPU p95", "value": "0.0216 ms", "note": "" }, { "label": "CUDA p95", "value": "0.1841 ms", "note": "" }, { "label": "CUDA training time", "value": "12.157 s", "note": "selected epoch 3" }, { "label": "Parameters", "value": "1,573", "note": "model size 50,765 bytes" } ], "ood": { "title": "Two independently frozen windows", "train_counts": "{0, 3, 6, 9}", "ood_counts": "{1, 2, 4, 5, 7, 8, 12}", "window": "500 cells, generated separately, no retraining", "metrics": [ { "label": "OOD accuracy", "value": "93.40%", "note": "" }, { "label": "OOD macro-F1", "value": "0.9260", "note": "" }, { "label": "Improvement over majority-class baseline", "value": "CI95 +0.514 to +0.612", "note": "" } ], "note": "The learned numeric policy survives a genuinely different count distribution and clears the two-independently-frozen-windows requirement." }, "teacher_rule": { "title": "The audit-of-the-audit", "body": "A 99.05% headline is meaningless until the student is shown to match a known teacher. The honest audit recovered the deterministic teacher policy from the dispatch guard.", "steps": [ "Select the provider with the lowest recent dispatch count.", "Break ties in priority order: MiniMax → Z.ai → Grok → Claude → Kimi." ], "result": "That rule scores 100% on the same 105-example test set.", "verdict": "The correct production artifact today is the explicit rule, not the model. The MLP is an excellent mimic of a simpler rule that runs in 20 lines of stdlib. Keep V5 as a shadow benchmark and revisit only if a future routing policy becomes noisy, weighted, or adaptive." }, "gates": [ { "gate": "Accuracy / macro-F1 over every baseline", "bar": "CI excludes 0", "actual": "99.05% / 0.9831 with accuracy CI95 97.14–100%", "pass": true }, { "gate": "All five providers have recall > 0", "bar": "yes", "actual": "104 of 105 correct across the five-class surface", "pass": true }, { "gate": "Strict validity rate", "bar": "≥ 95%", "actual": "100% — no generation step to be invalid", "pass": true }, { "gate": "p95 latency budget", "bar": "≤ 250 ms", "actual": "0.0773 ms on the stdlib CPU serving path", "pass": true }, { "gate": "Two independently frozen windows", "bar": "generalizes to unseen counter values", "actual": "93.40% / 0.9260 on a 500-cell OOD window", "pass": true }, { "gate": "Beats the recovered deterministic teacher", "bar": "strictly above the rule", "actual": "rule 100% vs model 99.05% — the rule wins", "pass": false } ], "verification": [ "64 / 64 targeted tests and 169 / 169 full-suite tests green.", "The job manifest was accepted on the first submission with no missing-key errors.", "Torch and stdlib forward passes produced identical predictions on all 105 test rows; maximum logit difference 3.09e-7.", "The GPU job completed in 33.122 s with no timeout, tracker error, or capacity issue.", "Artifact modes enforced: 0600 on the model JSON and the private evaluation, 0664 on the public evaluation.", "Public eval, private eval and model JSON each carry a matching SHA-256 in the receipt's provenance list.", "The experiment tracker records the run with status success." ], "not_closed": [ "It does not produce a learned policy that beats the deterministic rule.", "It does not generalize to task classes other than background.", "It does not prove the rule is the best policy for any future routing change — only for the current production policy." ], "artifacts": [ { "name": "V5 public evaluation", "path": "tooling/artifacts/router-school-v5-fairness-mlp-eval.json", "visibility": "public", "sha256": "d4f0b8f52d000f8bbb8432a962db3e62cf238cc94e50dc1de94cad6e0f74590d", "note": "mode 0664 — public by design" }, { "name": "V5 private evaluation", "path": "tooling/artifacts/router-school-v5-fairness-mlp-eval-private.json", "visibility": "private", "sha256": null, "note": "mode 0600 — row-level bodies withheld; digest held in the private receipt" }, { "name": "V5 model", "path": "tooling/artifacts/models/router-v5-fairness-mlp.json", "visibility": "private", "sha256": null, "note": "mode 0600 — digest held in the private receipt" }, { "name": "V5 OOD window", "path": "tooling/artifacts/router-school-v5-ood-window.json", "visibility": "public", "sha256": null, "note": "a private companion window is held at mode 0600" }, { "name": "V5 compute receipt", "path": "gpu-lab/receipts/20260804T144750Z-router-school-v5-fairness-mlp.json", "visibility": "public", "sha256": null, "note": "provenance.artifacts carries the matching digest for every artifact above" }, { "name": "Experiment-tracker run", "path": "mlflow run 7c1d3fdd4470470eba842f7bab883ec2", "visibility": "public", "sha256": null, "note": "status success" } ] } ], "next_questions": [ { "id": "v6-noisy-regime", "title": "Tiny MLP on an intentionally noisy policy", "body": "The current teacher is deterministic, which is exactly why the rule wins. Add a slight bias toward a preferred provider and the rule stops being exactly recoverable — that creates a real target for a learned model rather than a mimicry exercise.", "rung": "ladder rung 3, re-run against a policy worth learning" }, { "id": "v6-boosted-trees", "title": "Boosted trees on the same features", "body": "Rung 4 of the classification ladder. Worth running if the MLP plateaus on a harder regime, since boosted trees are frequently the best accuracy/latency tradeoff on tabular inputs.", "rung": "ladder rung 4" }, { "id": "v6-rule-shadow", "title": "Shadow the rejected cells against the rule", "body": "Re-score the V4 generative and V4-1B cells with the recovered rule as the baseline, to confirm the rule wins on every count distribution rather than only on the frozen test window.", "rung": "regression check on rung 1" }, { "id": "v6-task-class-breadth", "title": "Beyond the background task class", "body": "V5 is a fairness specialist on a single task class. Nothing in this program yet shows that a numeric policy learner generalizes across the ten-class task surface that V3 sampled too thinly.", "rung": "curriculum scope" } ] }