{ "schema": "router-school.methodology.v1", "last_updated": "2026-08-04", "falsification_gates": { "title": "The falsification gate set", "lede": "A shadow-eval delta is never evidence by itself. Even with a proper chronological holdout and a frozen baseline, a headline number can be a trivial-rule match, leakage, or majority-class collapse. These are the checks a cell has to survive before the word promote is allowed.", "core_set": "{ leakage, rule_parity, macro_f1_delta, bootstrap_ci_excludes_zero, mcnemar_p_lt_0.05 }", "gates": [ { "n": 1, "id": "rule_parity", "name": "Trivial-rule parity ceiling", "bar": "The cell's macro-F1 must exceed a five-line task_class → lane lookup's macro-F1 by a margin that survives the bootstrap CI.", "why": "This is the trap V1 fell into. A lookup table fitted by majority vote per class can reproduce a +5 pp accuracy headline exactly. Run it before reporting any positive delta.", "evidence": "V1: adapter 0.326 vs rule 0.326 — bit-identical." }, { "n": 2, "id": "leakage", "name": "Leakage detection", "bar": "No prompt body in test may appear anywhere in train — byte compare plus hashed Jaccard. Report the raw duplicate count and the accuracy split between duplicate and clean rows.", "why": "Memorized rows print as skill. Dedup must be part of the split itself, not a post-hoc filter, or the split silently reintroduces it on the next rebuild.", "evidence": "V1: 46/98 test bodies collided with train, 22/98 verbatim. The honest delta on the 76 clean rows was ≈ +2.6 pp, not +5.10." }, { "n": 3, "id": "macro_f1_delta", "name": "Macro-F1 over accuracy", "bar": "Report cell macro-F1 and trivial-rule macro-F1 side by side. If the difference is noise, the accuracy delta is noise too.", "why": "Accuracy is dominated by majority-class collapse; macro-F1 is class-balanced. A model that defaults to the most common training class prints positive accuracy deltas while never predicting some classes at all.", "evidence": "V1: minimax predicted 72/98 times, claude recall ≈ 0.04, zai never predicted at all." }, { "n": 4, "id": "bootstrap_ci_excludes_zero", "name": "Bootstrap CI excludes zero", "bar": "Resample the per-example correct/incorrect vector with replacement (N = 10,000) and require the CI95 on the accuracy delta to exclude zero.", "why": "A ±5 pp delta at n ≈ 100 has a confidence interval wide enough to span zero even when the point estimate looks clean.", "evidence": "V1: CI95 was [−8.2, +18.4] pp." }, { "n": 5, "id": "mcnemar_p_lt_0.05", "name": "McNemar exact, two-sided", "bar": "p < 0.05 on the discordant pairs.", "why": "The paired test is what distinguishes a real per-example improvement from a reshuffle of the same error budget.", "evidence": "V1: b=20, c=25 → p ≈ 0.55." }, { "n": 6, "id": "no_hardcoded_numbers", "name": "No hardcoded numbers", "bar": "Grep the evaluator source for any literal float that could reach the report. Every baseline must be recomputed from real inference on the eval set.", "why": "A frozen baseline that is frozen because it is a constant is not a baseline.", "evidence": "V1: the 47.96% baseline was a literal in the eval runner source." }, { "n": 7, "id": "no_zero_recall_class", "name": "No zero-recall class", "bar": "Every provider class must have recall > 0.", "why": "Class collapse is the failure mode that a single aggregate metric is least able to see.", "evidence": "V2 E4 collapsed to 0.00% accuracy after an 80× oversample of the rarest class." }, { "n": 8, "id": "strict_validity", "name": "Strict schema validity ≥ 95%", "bar": "Output must validate against a schema whose enum values are derived from the corpus, not hardcoded — for the routing task, exactly four keys.", "why": "A permissive validity check that accepts any parseable dict measures nothing.", "evidence": "V1 accepted any dict. V2 tightened it to exactly four enum-checked keys." }, { "n": 9, "id": "latency_budget", "name": "p95 latency ≤ 250 ms", "bar": "Measured on the actual serving path, not on the training device.", "why": "A correct decision that cannot be made inside the request budget is not a routing policy. A latency violation is a structural signal to re-derive the architecture, not a prompt to shrink the model.", "evidence": "V4 measured 1,635.9 ms at 3B and 954.08 ms at 1B. V5 measures 0.0773 ms." }, { "n": 10, "id": "two_frozen_windows", "name": "Two independently frozen windows", "bar": "A second evaluation window generated separately, with input values never seen in train, scored without retraining.", "why": "One frozen holdout proves the split was honest. Two prove the policy generalizes rather than interpolating the sampled grid.", "evidence": "V5: train counts {0, 3, 6, 9}, OOD window counts {1, 2, 4, 5, 7, 8, 12} → 93.40% / 0.9260." }, { "n": 11, "id": "beats_recovered_teacher", "name": "Beats the recovered teacher rule", "bar": "Recover the deterministic policy the training labels came from, score it on the same test set, and require the learned model to beat it.", "why": "The audit-of-the-audit. A learned classifier that matches a deterministic teacher at 100% is not a shipping candidate — it is evidence the architectural ladder was the wrong choice and the rule is the actual artifact.", "evidence": "V5: the recovered rule scores 100% against the model's 99.05%. The rule stays in production." } ] }, "baselines": { "title": "Three baselines, not one", "lede": "V1's recipe called for a single frozen deterministic router. Every cell since requires three, because each answers a different falsification question and the audit checks all three at once.", "rows": [ { "id": "majority_class", "question": "Is the cell learning anything at all?", "why": "The trap is class collapse. A model that always predicts the most common class prints a positive accuracy delta on a skewed corpus. This is the lower bound, and the cell must beat it on macro-F1, not accuracy." }, { "id": "task_class_rule", "question": "Did the cell learn the trivial rule?", "why": "The structural trap V1 fell into. A task_class → mode lookup fits in five lines. This is the parity ceiling the cell must clear." }, { "id": "tfidf_logreg", "question": "Is the learnable signal in the features?", "why": "A balanced sparse linear model answers whether there is learnable structure at all. If it beats the trained cell on macro-F1, the cell is failing to find structure a linear model already finds. Verified on V3: balanced TF-IDF reached macro-F1 0.7559 with 100% recall on the rarest class." } ], "rule": "Every matched-pairs evaluation emits all three under stable paths, with SHA-256 digests recorded in the manifest before the GPU job is submitted. The eval receipt's baseline block then carries three measurement anchors, and the audit has three arithmetic deltas to recompute against independent files." }, "audit_pattern": { "title": "Two independent auditors", "points": [ "Dispatch at least two audit lanes from different model families, with read-only filesystem tools and a prompt that says recompute every claim from the raw artifacts, do not trust the parent's summary.", "A single auditor's 'looks plausible' is a single point of failure. Two auditors landing on the same wrong number means the verifier is broken, not the result.", "The convergence is the proof. In V1 a 337-line audit and a 257-line audit reached identical finding sets without seeing each other.", "When gates fail, correct the public record before doing anything else. Not 'rerun with tweaked hyperparameters', not 'ship anyway with caveats'." ] }, "discipline": { "title": "Standing rules", "rules": [ { "name": "Preregistration before training", "body": "Dataset disclosure, metric set, baseline set, promotion gates and disallowed behaviors are written before a GPU job is submitted. When a planned baseline turns out to be unavailable, the substitution and its exact score go into the preregistration before the harness ships — a silent baseline swap is what an audit is for." }, { "name": "Strict RED-GREEN-REFACTOR", "body": "Every new behavior gets a failing test first, and the failing test has to fail for the right reason — feature missing, not a typo or a bad import path." }, { "name": "Suspect the evaluator first", "body": "When accuracy and schema validity both read zero and every baseline delta is negative with probability 1.0, the first hypothesis is a prompt-template or harness mismatch. The second is the adapter. The third is the dataset. Order matters: an evaluator misdiagnosis costs redundant GPU hours and contaminates the publication record." }, { "name": "Falsification is a lesson, not a stop signal", "body": "Correct the record, then iterate. A killed cell publishes as its own outcome and never overrides a prior audit's verdict." }, { "name": "Public receipts carry no private text", "body": "Raw generations live only in mode-0600 private artifacts. The public artifact strips prompts and completions and keeps the metrics, the provenance and the digests." } ] } }