{ "run_id": "2026-09-30", "date": "2026-09-30", "entrants": [ { "id": "claude-code-fable", "name": "Claude Code (Fable 5.1)", "model": "Claude Fable 5.1", "reasoning": "High", "client_version": "2.1.285 (Claude Code)", "notes": "Claude Code in headless mode, high reasoning effort, on a Claude subscription.", "short_model": "Fable 5.1", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 10, "cached_input": 0.25, "cache_write": 12.5, "cache_write_1h": 20, "output": 50 }, "model": "anthropic/claude-fable-5.1", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 13, "score": 65.0, "partial_score": 98.0, "graded": 20, "total_api_equiv_usd": 75.182298, "unpriced_attempts": 0, "api_equiv_usd_per_pass": 5.783254 }, { "id": "claude-code-opus", "name": "Claude Code (Opus 5.5)", "model": "Claude Opus 5.5", "reasoning": "High", "client_version": "2.1.285 (Claude Code)", "notes": "Claude Code in headless mode, high reasoning effort, on a Claude subscription.", "short_model": "Opus 5.5", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 4, "cached_input": 0.2, "cache_write": 5, "cache_write_1h": 8, "output": 20 }, "model": "anthropic/claude-opus-5.5", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 19, "score": 95.0, "partial_score": 99.7, "graded": 20, "total_api_equiv_usd": 30.770923, "unpriced_attempts": 0, "api_equiv_usd_per_pass": 1.619522 }, { "id": "claude-code-sonnet", "name": "Claude Code (Sonnet 5.5)", "model": "Claude Sonnet 5.5", "reasoning": "High", "client_version": "2.1.285 (Claude Code)", "notes": "Claude Code in headless mode, high reasoning effort, on a Claude subscription.", "short_model": "Sonnet 5.5", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 2, "cached_input": 0.2, "cache_write": 2.5, "cache_write_1h": 4, "output": 10 }, "model": "anthropic/claude-sonnet-5.5", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 17, "score": 85.0, "partial_score": 99.2, "graded": 20, "total_api_equiv_usd": 16.931636, "unpriced_attempts": 0, "api_equiv_usd_per_pass": 0.995979 }, { "id": "codex-astra", "name": "Codex (GPT-6 Astra)", "model": "GPT-6 Astra", "reasoning": "High", "client_version": "codex-cli 0.159.2", "notes": "Codex CLI (exec mode), high reasoning effort, on a ChatGPT subscription.", "short_model": "GPT-6 Astra", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 10, "cached_input": 1, "cache_write": 12.5, "output": 50 }, "model": "openai/gpt-6-astra", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 17, "score": 85.0, "partial_score": 98.6, "graded": 20, "total_api_equiv_usd": 29.787396, "unpriced_attempts": 0, "api_equiv_usd_per_pass": 1.7522 }, { "id": "codex-sol", "name": "Codex (ChatGPT Sol 6.1)", "model": "ChatGPT Sol 6.1", "reasoning": "High", "client_version": "codex-cli 0.159.2", "notes": "Codex CLI (exec mode), high reasoning effort, on a ChatGPT subscription.", "short_model": "Sol 6.1", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 2, "cached_input": 0.1, "cache_write": 2.5, "output": 10 }, "model": "openai/gpt-6.1-sol", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 15, "score": 75.0, "partial_score": 98.6, "graded": 20, "total_api_equiv_usd": 5.23801, "unpriced_attempts": 2, "api_equiv_usd_per_pass": 0.349201 }, { "id": "grok-build", "name": "Grok Build", "model": "Grok 4.7", "reasoning": "High", "client_version": "grok 1.0.44 (5b807183dd79)", "notes": "Grok Build CLI in headless mode, high reasoning effort, on a SuperGrok subscription.", "short_model": "Grok 4.7", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 2, "cached_input": 0.5, "output": 6 }, "model": "x-ai/grok-4.7", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 9, "score": 45.0, "partial_score": 75.5, "graded": 20, "total_api_equiv_usd": 34.41168, "unpriced_attempts": 0, "api_equiv_usd_per_pass": 3.82352 }, { "id": "muse-code", "name": "Muse Code", "model": "Muse Spark 1.3", "reasoning": "High", "client_version": "Muse Code 1.4.1 (1.4.1-R4503.1)", "notes": "Muse Code CLI (exec mode), high reasoning effort, on a Muse Code subscription. Meta publishes no per-token price, so cost is unknown.", "short_model": "Muse Spark 1.3", "cost_basis": "api-equivalent (subscription)", "prices": { "per_million_tokens_usd": { "input": 1.25, "cached_input": 0.15, "output": 4.25 }, "model": "meta/muse-spark-1.3", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 11, "score": 55.0, "partial_score": 90.6, "graded": 20, "total_api_equiv_usd": null, "unpriced_attempts": 20, "api_equiv_usd_per_pass": null }, { "id": "opencode-openrouter", "name": "OpenCode (OpenRouter)", "model": "DeepSeek V4.1 Flash", "reasoning": "High", "client_version": "1.18.33", "notes": "OpenCode via OpenRouter, served by DeepInfra (US), high reasoning effort. Cost is the real charge.", "short_model": "DeepSeek V4.1 Flash", "cost_basis": "api-billed", "prices": { "per_million_tokens_usd": { "input": 0.3, "cached_input": 0.006, "output": 1.2 }, "model": "deepseek/deepseek-v4.1-flash", "checked": "2026-09-30", "source": "https://openrouter.ai/api/v1/models" }, "attempts": 20, "passes": 14, "score": 70.0, "partial_score": 94.7, "graded": 20, "total_api_equiv_usd": 2.770533, "unpriced_attempts": 0, "api_equiv_usd_per_pass": 0.197895 } ], "tasks": [ { "id": "cad-enclosure", "category": "cad", "title": "Parametric sensor-board enclosure in OpenSCAD", "summary": "Model a two-part electronics enclosure (base and lid) in OpenSCAD from a board drawing and a spec, with connector cutouts, standoffs, a fitted lid and fully parametric dimensions.", "grading": "tests" }, { "id": "cad-sheet-metal", "category": "cad", "title": "Laser-cut flat pattern for a formed bracket", "summary": "Turn a formed steel bracket (dimensions and drawing) into a DXF flat pattern with bend lines and a bend table, using the given thickness, bend radius and K-factor.", "grading": "tests" }, { "id": "data-experiment-analysis", "category": "data", "title": "Analyse a checkout A/B test from raw event logs", "summary": "From a raw event log, clean and analyse an A/B test of a new checkout (duplicates, bots skewing the split, device mix, a week-1 novelty effect) and give exact figures, the SRM p-value, a decision, and an honest write-up.", "grading": "judged" }, { "id": "data-month-end", "category": "data", "title": "October month-end: bank, ledger and payment processor", "summary": "Reconcile a bank export, an accounting-system export and a payment processor's payouts across month-end (UTC vs Melbourne time with daylight saving), and report exact figures and every unreconciled item with its reason.", "grading": "tests" }, { "id": "design-from-inspiration", "category": "design", "title": "Design a landing page that borrows an inspiration image's visual language", "summary": "Build an RS9 service landing page from supplied content and fonts that clearly reuses the palette, type pairing, hard-edged components and layout motifs of an inspiration screenshot, without copying its content.", "grading": "judged" }, { "id": "design-recreate-mockup", "category": "design", "title": "Rebuild a landing page from desktop, tablet and mobile mockups", "summary": "Recreate a marketing landing page from four PNG mockups (desktop, tablet, mobile, open mobile menu) with supplied brand tokens and fonts: responsive, accessible, and measured against the mockups.", "grading": "judged" }, { "id": "documents-contract-changes", "category": "documents", "title": "Apply 14 agreed changes across a contract set", "summary": "Fourteen negotiated changes (renumbered clauses and schedules, compounding price changes, moved dates) must be applied across an MSA, SOW, three Schedules and a Price Book, with every cross-reference and total kept right.", "grading": "tests" }, { "id": "documents-tender-response", "category": "documents", "title": "Answer a council tender from the supplier's own documents", "summary": "Respond to a 30-requirement council RFT with a compliance matrix and word-limited sections, using only facts that RS9's own documents support, including a priced five-year offer worked from several sources.", "grading": "judged" }, { "id": "frontier-cad-assembly", "category": "cad", "title": "Hinged sensor clamp and wall bracket, measured from a vendor STL", "summary": "Design a three-part OpenSCAD assembly (wall bracket, hinged clamp, hinge pin) around a vendor sensor STL that must be measured, with mating clearances, aligned bolt holes, a screw stack-up, wall and stiffness minimums, a print bed limit and no interference through a range of hinge rotation.", "grading": "tests" }, { "id": "frontier-codebase-change", "category": "software", "title": "Move a billing codebase from floats to integer minor units", "summary": "A 125-file invoicing package must switch all money from floats to integer minor units across domain code, SQLite storage with a data migration, the JSON API, CSV import and export and reports, following a finance policy, ADRs and an issue tracker export.", "grading": "tests" }, { "id": "frontier-tender-pricing", "category": "documents", "title": "Price a fabrication and installation tender from a messy pack", "summary": "Take off quantities from DXF drawings, cost them from supplier quotes and RS9's rate card and pricing policy, and deliver an exact filled schedule of rates with GST plus a short pricing memo.", "grading": "judged" }, { "id": "frontier-web-app", "category": "interfaces", "title": "Build a three-page hardware shop from mockups and a behaviour spec", "summary": "Build a filterable catalogue with URL state and history, a product page and a localStorage cart with pricing rules, in plain HTML/CSS/JS, matching desktop and mobile mockups and passing hidden browser behaviour tests.", "grading": "judged" }, { "id": "handoff-long-thread", "category": "handoff", "title": "Finish a feature after a five-day chat thread of changing requirements", "summary": "Take over a half-built trade quote calculator, work out the currently agreed rules from a 350-message chat export full of reversals, a side-thread decision and an unconfirmed idea, finish it, and write a handoff citing the deciding messages.", "grading": "judged" }, { "id": "handoff-multi-cause-incident", "category": "handoff", "title": "Diagnose a two-cause checkout incident across three services", "summary": "From about 14,000 lines of logs in three time formats, metrics, deploy history and config diffs, find the two changes that combined into a retry storm (and dismiss the deploy that only looks guilty), name the first failure to the second, and propose a config fix that survives a simulated replay.", "grading": "tests" }, { "id": "interfaces-cut-list", "category": "interfaces", "title": "Bill of materials and optimal cut list from a revised drawing set", "summary": "Read text-based fabrication drawings with revisions, a draft, two change notes and a purchase order, produce a bill of materials, then an exact minimum-new-bar cut list with kerf that uses an offcut register.", "grading": "tests" }, { "id": "interfaces-quote-wizard", "category": "interfaces", "title": "Build a sheet-metal quote wizard to a 30-rule spec", "summary": "Build a plain HTML/CSS/JS multi-step quote wizard with conditional steps, unit conversion, quantity tiers, GST and accessible errors, with the rules in a tested ES module.", "grading": "tests" }, { "id": "research-claims-audit", "category": "research", "title": "Audit a vendor's marketing claims against a pack of specs, tests and standards", "summary": "Label 20 marketing claims for a home battery as supported, contradicted or not determinable, naming the governing source and key figure, using 49 documents that include datasheets across revisions, lab reports, changelogs, a standard, a certification register and forum noise.", "grading": "tests" }, { "id": "research-grant-portfolio", "category": "research", "title": "Six clients, one grant: eligibility and maximum funding across amendments", "summary": "Work out eligibility and the exact maximum grant for six clients under a state grant whose rules were changed by three amendments, a correction and two ministerial notices, from a pack of 39 documents that includes conflicting FAQs and outdated summaries.", "grading": "judged" }, { "id": "software-checkout-bugs", "category": "software", "title": "Fix wrong totals in a checkout service", "summary": "A Python e-commerce checkout has many interacting bugs behind a support ticket about wrong totals: GST rounding, promo dates across Melbourne daylight saving, a stale price cache, refunds, and the places that repeat the same sums. Fix them to the finance rules without breaking the existing tests.", "grading": "tests" }, { "id": "software-migration", "category": "software", "title": "Add multi-location stock with a safe data migration", "summary": "Add multi-location stock to a live SQLite-backed stock service: migrate the production data exactly and idempotently, keep the old API byte-for-byte compatible, and add locations, transfers and per-location reports.", "grading": "tests" } ], "attempts": [ { "entrant": "claude-code-fable", "task": "cad-enclosure", "status": "pass", "duration_s": 338.7, "cost_usd": null, "checks": [ { "id": "file", "desc": "enclosure.scad exists", "pass": true }, { "id": "params-declared", "desc": "The variables part, wall, floor_t, standoff_h and fit are declared", "pass": true, "note": "missing top-level variables: []" }, { "id": "render-base", "desc": "OpenSCAD renders part=base", "pass": true }, { "id": "render-lid", "desc": "OpenSCAD renders part=lid", "pass": true }, { "id": "base-watertight", "desc": "Base is one watertight, manifold solid", "pass": true, "note": "watertight=True vol=30225" }, { "id": "base-bbox", "desc": "Base bounding box is the expected outer size, min corner at the origin (0.2)", "pass": true, "note": "got [0. 0. 0.] .. [85.5 57.6 28. ], want max 85.50,57.60,28.0" }, { "id": "base-rounding", "desc": "Outer vertical corners R3.0 and bottom edge fillet R1.5 (slices at the rim and near the bottom)", "pass": true }, { "id": "cavity", "desc": "Inside cavity is the board-derived rectangle (1.0 clearance, barrel jack overhang on the right)", "pass": true, "note": "inner bbox x 2.40..83.10 y 2.40..55.20; want x 2.4..83.10 y 2.4..55.20" }, { "id": "wall-thickness", "desc": "Wall thickness is 2.4 on all four sides (ray casts at several heights)", "pass": true, "note": "0/127 bad samples, e.g. []" }, { "id": "floor", "desc": "Floor is 2.0 thick and closed underneath", "pass": true, "note": "[]" }, { "id": "standoff-positions", "desc": "Four standoffs, OD 6.0, at the hole positions read from the drill file (slice at mid-height)", "pass": true, "note": "missing or misplaced/wrong-size standoffs: " }, { "id": "pilot-holes", "desc": "Each standoff has a dia 2.5 pilot hole, centred", "pass": true, "note": "pilot hole (dia 2.5) missing at " }, { "id": "no-extra-posts", "desc": "No standoff at the test-point hole, and nothing else in the cavity at standoff height", "pass": true, "note": "14 loops at standoff mid-height (want 14), 0 near H5" }, { "id": "lid-posts", "desc": "Four corner screw posts: OD 7.0, 5.0 from both inside wall faces, top flush at 28.0, pilot 2.5 x 8.0 deep", "pass": true }, { "id": "standoff-height", "desc": "Standoff tops are `standoff_h` above the floor", "pass": true, "note": "top z wrong (want 8.0): []" }, { "id": "pilot-depth", "desc": "Pilot holes are blind, 5.0 deep from the standoff top, floor underneath intact", "pass": true, "note": "want blind hole bottom at 3.00 and closed floor: []" }, { "id": "cutouts", "desc": "All five connector openings: right face, right place and height, right size, through the whole wall; no others", "pass": true }, { "id": "lid-watertight", "desc": "Lid is one watertight, manifold solid", "pass": true, "note": "watertight=True" }, { "id": "lid-bbox", "desc": "Lid bounding box: full outer rectangle, Z 24.0 to 30.5, in assembled position (0.2)", "pass": true, "note": "got [ 0. 0. 24.] .. [85.5 57.6 30.5]" }, { "id": "lip-fit", "desc": "Lip outside is the cavity minus `fit` on each side, centred (0.06)", "pass": true, "note": "lip outer x 2.650..82.850 y 2.650..54.950; want x 2.650..82.850 y 2.650..54.950" }, { "id": "lip-wall", "desc": "Lip is a ring 1.6 thick", "pass": true, "note": "lip wall [1.6 1.6 1.6 1.6]" }, { "id": "lid-heights", "desc": "Plate rests on the rim (Z 28.0), is 2.5 thick, lip is 4.0 deep", "pass": true, "note": "[]" }, { "id": "lid-windows", "desc": "Windows over C12 (round) and DS1 (rectangle, rotated 90 degrees), sized part + 0.5 per side, at the placed centres", "pass": true }, { "id": "lid-holes", "desc": "Two LED holes (dia 3.0) and the hexagonal vent pattern (flat-to-flat 4.0, vertices on X) in the right places, nothing else", "pass": true, "note": "missing/wrong: [] (0 total)" }, { "id": "lid-outline", "desc": "Lid plate outline has R3.0 corners (same outline as the base)", "pass": true, "note": "outline area 4917.0, want 4917.1 (corner radius 3.0)" }, { "id": "lid-recess", "desc": "Label recess 30 x 10, 0.6 deep, centred at board-frame (30, 6)", "pass": true }, { "id": "lid-countersinks", "desc": "Four countersunk screw holes over the posts: dia 3.4 through, 90 degree countersink to dia 6.4 at the top", "pass": true }, { "id": "lip-scallops", "desc": "Lip is cut back around each post by exactly the fit clearance (radius post + fit), lip not otherwise cut", "pass": true }, { "id": "no-interference", "desc": "Assembled base and lid do not overlap in volume", "pass": true, "note": "overlap volume 0.00 mm3" }, { "id": "param-render-base", "desc": "OpenSCAD renders part=base [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-render-lid", "desc": "OpenSCAD renders part=lid [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-base-watertight", "desc": "Base is one watertight, manifold solid [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "watertight=True vol=37114" }, { "id": "param-base-bbox", "desc": "Base bounding box is the expected outer size, min corner at the origin (0.2) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "got [0. 0. 0.] .. [86.7 58.8 28. ], want max 86.70,58.80,28.0" }, { "id": "param-base-rounding", "desc": "Outer vertical corners R3.0 and bottom edge fillet R1.5 (slices at the rim and near the bottom) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-cavity", "desc": "Inside cavity is the board-derived rectangle (1.0 clearance, barrel jack overhang on the right) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "inner bbox x 3.00..83.70 y 3.00..55.80; want x 3.0..83.70 y 3.0..55.80" }, { "id": "param-wall-thickness", "desc": "Wall thickness is 3.0 on all four sides (ray casts at several heights) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "0/122 bad samples, e.g. []" }, { "id": "param-floor", "desc": "Floor is 2.5 thick and closed underneath [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "[]" }, { "id": "param-standoff-positions", "desc": "Four standoffs, OD 6.0, at the hole positions read from the drill file (slice at mid-height) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "missing or misplaced/wrong-size standoffs: " }, { "id": "param-pilot-holes", "desc": "Each standoff has a dia 2.5 pilot hole, centred [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "pilot hole (dia 2.5) missing at " }, { "id": "param-no-extra-posts", "desc": "No standoff at the test-point hole, and nothing else in the cavity at standoff height [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "14 loops at standoff mid-height (want 14), 0 near H5" }, { "id": "param-lid-posts", "desc": "Four corner screw posts: OD 7.0, 5.0 from both inside wall faces, top flush at 28.0, pilot 2.5 x 8.0 deep [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-standoff-height", "desc": "Standoff tops are `standoff_h` above the floor [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "top z wrong (want 10.5): []" }, { "id": "param-pilot-depth", "desc": "Pilot holes are blind, 5.0 deep from the standoff top, floor underneath intact [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "want blind hole bottom at 5.50 and closed floor: []" }, { "id": "param-cutouts", "desc": "All five connector openings: right face, right place and height, right size, through the whole wall; no others [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-lid-watertight", "desc": "Lid is one watertight, manifold solid [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "watertight=True" }, { "id": "param-lid-bbox", "desc": "Lid bounding box: full outer rectangle, Z 24.0 to 30.5, in assembled position (0.2) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "got [ 0. 0. 24.] .. [86.7 58.8 30.5]" }, { "id": "param-lip-fit", "desc": "Lip outside is the cavity minus `fit` on each side, centred (0.06) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "lip outer x 3.400..83.300 y 3.400..55.400; want x 3.400..83.300 y 3.400..55.400" }, { "id": "param-lip-wall", "desc": "Lip is a ring 1.6 thick [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "lip wall [1.6 1.6 1.6 1.6]" }, { "id": "param-lid-heights", "desc": "Plate rests on the rim (Z 28.0), is 2.5 thick, lip is 4.0 deep [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "[]" }, { "id": "param-lid-windows", "desc": "Windows over C12 (round) and DS1 (rectangle, rotated 90 degrees), sized part + 0.5 per side, at the placed centres [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-lid-holes", "desc": "Two LED holes (dia 3.0) and the hexagonal vent pattern (flat-to-flat 4.0, vertices on X) in the right places, nothing else [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "missing/wrong: [] (0 total)" }, { "id": "param-lid-outline", "desc": "Lid plate outline has R3.0 corners (same outline as the base) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "outline area 5090.2, want 5090.2 (corner radius 3.0)" }, { "id": "param-lid-recess", "desc": "Label recess 30 x 10, 0.6 deep, centred at board-frame (30, 6) [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-lid-countersinks", "desc": "Four countersunk screw holes over the posts: dia 3.4 through, 90 degree countersink to dia 6.4 at the top [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-lip-scallops", "desc": "Lip is cut back around each post by exactly the fit clearance (radius post + fit), lip not otherwise cut [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true }, { "id": "param-no-interference", "desc": "Assembled base and lid do not overlap in volume [re-rendered with wall=3.0 floor_t=2.5 standoff_h=8.0 fit=0.4]", "pass": true, "note": "overlap volume 0.00 mm3" } ], "judge": null, "note": "", "startup_s": 1.4, "cost_basis": "api-equivalent (subscription)", "tokens": { "input": 418, "cached_input": 566105, "cache_write": 0, "cache_write_1h": 50691, "output": 24762, "reasoning": 6827 }, "api_equiv_usd": 2.397626, "usage_note": "", "api_equiv_check_usd": 2.397626249999999 }, { "entrant": "claude-code-fable", "task": "cad-sheet-metal", "status": "pass", "duration_s": 104.1, "cost_usd": null, "checks": [ { "id": "dxf-exists", "desc": "flat.dxf exists and reads with ezdxf", "pass": true }, { "id": "units", "desc": "DXF is in millimetres (or unitless) and holds only CUT and BEND geometry", "pass": true, "note": "units=4 other=[]" }, { "id": "closed", "desc": "CUT geometry forms closed loops only (closed outline, no open or stray pieces)", "pass": true, "note": "0 open chains, 9 loops" }, { "id": "outline-single", "desc": "Exactly one outer outline, with every other loop inside it", "pass": true }, { "id": "flat-size", "desc": "Blank is 147.324 x 50.0 (0.1) with the toe free edge at X=0 and datum edge at Y=0", "pass": true, "note": "got 0.000..147.324 x 0.000..50.000, want 0..147.324 x 0..50.0" }, { "id": "outline-corners", "desc": "Outline area matches 5x5 chamfers on the toe end and R4 corners on the lip end (4 mm2)", "pass": true, "note": "area 7334.3, want 7334.3" }, { "id": "holes-f1", "desc": "F1 holes (2 x dia 5.5): centres and diameters", "pass": true }, { "id": "holes-base", "desc": "Base holes (2 x dia 6.6): centres and diameters", "pass": true }, { "id": "holes-f2", "desc": "F2 hole (dia 8.0): centre and diameter", "pass": true }, { "id": "holes-lip", "desc": "Lip holes (2 x dia 4.5): centres and diameters", "pass": true }, { "id": "slot", "desc": "Base slot 16 x 6 obround, long axis across the part, centre at the right place", "pass": true, "note": "slot bbox 16.00x6.00 area 88.2, want 16.0x6.0 area 88.3" }, { "id": "hole-count", "desc": "Exactly 8 inner cut-outs (7 round holes and 1 slot): none missing, none extra", "pass": true, "note": "8 inner loops" }, { "id": "bend-lines", "desc": "Four full-width bend lines on layer BEND at the neutral-axis centres (0.1)", "pass": true, "note": "bend lines at [14.199, 38.414, 98.446, 135.893], want [14.199, 38.414, 98.446, 135.893]" }, { "id": "csv-shape", "desc": "bend_table.csv has the exact header and 4 rows", "pass": true, "note": "header ['bend', 'position_mm', 'direction', 'angle_deg', 'inside_radius_mm', 'k_factor', 'bend_allowance_mm', 'outside_setback_mm'], rows 4" }, { "id": "csv-positions", "desc": "Bend table positions match the bend lines (0.1) and order", "pass": true, "note": "[14.199, 38.414, 98.446, 135.893] want [14.199, 38.414, 98.446, 135.893]" }, { "id": "csv-directions", "desc": "Bend directions: toe bend down, the other three up", "pass": true, "note": "['down', 'up', 'up', 'up']" }, { "id": "csv-angles-inputs", "desc": "Bend table angles 90/90/90/45, radii 2/3/3/4, K 0.40/0.42/0.42/0.44", "pass": true, "note": "[90.0, 90.0, 90.0, 45.0] [2.0, 3.0, 3.0, 4.0] [0.4, 0.42, 0.42, 0.44]" }, { "id": "csv-allowance", "desc": "Bend allowances use each bend's own radius and K (4.398, 6.032, 6.032, 3.833; 0.02)", "pass": true, "note": "[4.398, 6.032, 6.032, 3.833] want [4.398, 6.032, 6.032, 3.833]" }, { "id": "csv-setback", "desc": "Outside setbacks are 4.000, 5.000, 5.000, 2.485 (0.02)", "pass": true, "note": "[4.0, 5.0, 5.0, 2.485] want [4.0, 5.0, 5.0, 2.485]" } ], "judge": null, "note": "", "startup_s": 0.8, "cost_basis": "api-equivalent (subscription)", "tokens": { "input": 98, "cached_input": 77158, "cache_write": 0, "cache_write_1h": 19003, "output": 8894, "reasoning": 4984 }, "api_equiv_usd": 0.845029, "usage_note": "", "api_equiv_check_usd": 0.8450295000000001 }, { "entrant": "claude-code-fable", "task": "data-experiment-analysis", "status": "pass", "duration_s": 167.2, "cost_usd": null, "checks": [ { "id": "shape", "desc": "experiment.json exists with the required keys", "pass": true }, { "id": "dup-rows", "desc": "Duplicate event rows and double-fired order rows counted correctly", "pass": true }, { "id": "excl-bots", "desc": "Bot users found (user agent and scraper IP range)", "pass": true }, { "id": "excl-others", "desc": "Internal, outside-window and contaminated users counted in the right order", "pass": true }, { "id": "srm-before-n", "desc": "SRM before exclusions: user counts per arm (case-variant ids merged)", "pass": true }, { "id": "srm-before-p", "desc": "SRM before exclusions: p-value to 3 decimals", "pass": true }, { "id": "srm-after-n", "desc": "SRM after exclusions: user counts per arm", "pass": true }, { "id": "srm-after-p", "desc": "SRM after exclusions: p-value to 3 decimals", "pass": true }, { "id": "overall-counts", "desc": "Overall users and converters per arm (24h window incl. boundary cases)", "pass": true }, { "id": "overall-uplift", "desc": "Overall pooled conversion rates and uplift", "pass": true }, { "id": "device-counts", "desc": "Per-device users and converters per arm", "pass": true }, { "id": "cov-balance", "desc": "Covariate-balance chi-square p-values for device, country and customer type", "pass": true }, { "id": "simpson", "desc": "Simpson's paradox flag is true", "pass": true }, { "id": "week1", "desc": "Week-1 standardised uplift and the novelty flag", "pass": true }, { "id": "steady-n", "desc": "Steady-state users per arm", "pass": true }, { "id": "steady-uplift", "desc": "Steady-state standardised uplift and 95% interval", "pass": true }, { "id": "revenue", "desc": "Revenue per user by arm (distinct orders, 24h window)", "pass": true }, { "id": "conclusion", "desc": "Conclusion from the fixed set follows the decision rules", "pass": true }, { "id": "report-length", "desc": "report.md exists and is about 150-250 words", "pass": true } ], "judge": { "model": "z-ai/glm-5.3", "items": [ { "n": 1, "pass": true, "reason": "JSON conclusion is invalid_rerun and the report's headline says the test is not valid, do not roll out B yet, fix the assignment and re-run; it declares B neither winner nor loser and never tells the owner to revert." }, { "n": 2, "pass": true, "reason": "Report says the 50/50 head-count is balanced after cleaning (3,126 vs 3,181) but the assignment cannot be trusted because B is far more mobile (78% vs 46%); bots are listed among the problems removed, and it neither says the split is simply fine nor attributes invalidity to the bot skew alone." }, { "n": 3, "pass": true, "reason": "Report explains the pooled comparison is misleading because B's arm is far more mobile and mobile converts less, so B looks worse overall (30.0% vs 28.0%) yet converts better within every device (desktop, mobile, tablet)." }, { "n": 4, "pass": true, "reason": "Report identifies the week-1 lift (~15 points) as likely novelty with the steady-state adjusted difference at 0.7 points (range -2.0 to +3.4, 'consistent with no lasting effect'), and never presents the week-1 or full-period lift as B's true effect." }, { "n": 5, "pass": true, "reason": "No causal claims as established fact ('appears', 'looks like', 'consistent with' throughout), and it states the arms were not comparable ('who landed in each arm is not [trustworthy]', 'the test is not valid'), so the adjusted figures are presented only as diagnostics, not a causal estimate." }, { "n": 6, "pass": true, "reason": "Every quoted figure matches the JSON: 30.0%/28.0% rates, 78%/46% mobile shares (2485/3181, 1433/3126), ~4-point adjusted uplift consistent with by_device rates, ~15 pts (14.69), 0.7 (0.68), -2.0/+3.4 (-1.99/3.36), 582/118 duplicates, 45 internal, 1,070 bots, 3,535/4,360 and 3,126/3,181." }, { "n": 7, "pass": true, "reason": "Written in plain English for a shop owner with a clear next step: fix the device-dependent assignment, check balance after two days, and run at least three more weeks." } ], "pass": true, "cost_usd": 0.118094 }, "note": "", "startup_s": 0.9, "cost_basis": "api-equivalent (subscription)", "tokens": { "input": 290, "cached_input": 289170, "cache_write": 0, "cache_write_1h": 29079, "output": 14295, "reasoning": 1842 }, "api_equiv_usd": 1.371522, "usage_note": "", "api_equiv_check_usd": 1.3715225 }, { "entrant": "claude-code-fable", "task": "data-month-end", "status": "pass", "duration_s": 281.4, "cost_usd": null, "checks": [ { "id": "shape", "desc": "month_end.json exists with the required keys and item fields", "pass": true }, { "id": "bank-close", "desc": "Bank closing balance at end of 31 Oct (Melbourne)", "pass": true }, { "id": "ledger-close", "desc": "Ledger closing balance (posted entries dated in October only)", "pass": true }, { "id": "difference", "desc": "Bank minus ledger difference", "pass": true }, { "id": "identity", "desc": "Difference equals the sum of the listed effects", "pass": true }, { "id": "proc-sales", "desc": "Processor gross sales, October Melbourne time, transactions counted once", "pass": true }, { "id": "proc-refunds", "desc": "Processor refunds", "pass": true }, { "id": "proc-disputes", "desc": "Processor net disputes (chargebacks less reversals)", "pass": true }, { "id": "proc-fees", "desc": "Processor fees over all transaction types", "pass": true }, { "id": "proc-net", "desc": "Processor net for the month", "pass": true }, { "id": "payout-count", "desc": "Number of paid payouts arriving in October Melbourne time (DST-aware)", "pass": true }, { "id": "payout-variances", "desc": "Payouts whose total differs from their transactions (manual adjustments) and by how much", "pass": true }, { "id": "payout-net", "desc": "Net of payouts arriving in October Melbourne time", "pass": true }, { "id": "held", "desc": "Balance PayLeaf still holds at month-end", "pass": true }, { "id": "items-exact", "desc": "Unreconciled list contains exactly the right bank/ledger items", "pass": true }, { "id": "items-recall", "desc": "Every genuine unreconciled item is listed", "pass": true }, { "id": "items-no-false", "desc": "Nothing is flagged that reconciles (renamed merchant, look-alikes, cleared opening items, voided entries)", "pass": true }, { "id": "reasons", "desc": "Every listed item has the right reason", "pass": true }, { "id": "effects", "desc": "Every listed item has the right signed effect", "pass": true }, { "id": "r-timing", "desc": "Timing items either side of month-end are identified", "pass": true }, { "id": "r-fx", "desc": "USD purchases with an FX difference are identified (and the exact-rate one is not)", "pass": true }, { "id": "r-dup", "desc": "Duplicate ledger entries are identified", "pass": true }, { "id": "r-groups", "desc": "Batch, split-payment and bank-fee groups are reported as single items", "pass": true }, { "id": "r-open", "desc": "The uncleared opening item is reported and cleared ones are not", "pass": true } ], "judge": null, "note": "", "startup_s": 0.3, "cost_basis": "api-equivalent (subscription)", "tokens": { "input": 258, "cached_input": 416830, "cache_write": 0, "cache_write_1h": 68113, "output": 25545, "reasoning": 12847 }, "api_equiv_usd": 2.746297, "usage_note": "", "api_equiv_check_usd": 2.7462975000000003 }, { "entrant": "claude-code-fable", "task": "design-from-inspiration", "status": "pass", "duration_s": 632.3, "cost_usd": null, "checks": [ { "id": "index-exists", "desc": "/work/index.html exists and renders", "pass": true }, { "id": "no-page-errors", "desc": "No JavaScript errors while loading", "pass": true }, { "id": "no-external", "desc": "No requests to the network (everything loads from /work)", "pass": true }, { "id": "no-hscroll-375", "desc": "No horizontal scrolling at 375 px wide", "pass": true }, { "id": "no-hscroll-1280", "desc": "No horizontal scrolling at 1280 px wide", "pass": true }, { "id": "viewport-meta", "desc": "Has a viewport meta tag", "pass": true }, { "id": "text-present", "desc": "All the brief's content is on the page, verbatim", "pass": true }, { "id": "no-copied-content", "desc": "None of the inspiration's content appears (no riso, zines, posters, Paste & Press, Brunswick, Fitzroy...)", "pass": true }, { "id": "no-hscroll-768", "desc": "No horizontal scrolling at 768 px wide", "pass": true }, { "id": "reflow-320", "desc": "Reflow (WCAG 1.4.10): no horizontal scrolling at 320 px wide, the width of a 1280 px screen zoomed to 400%", "pass": true }, { "id": "reflow-640-1920", "desc": "No horizontal scrolling at 640 px (200% zoom) or on a 1920 px wide screen", "pass": true }, { "id": "skip-link", "desc": "The first Tab stop is a visible 'skip to content' link that targets the main content", "pass": true }, { "id": "testids", "desc": "All required data-testid hooks exist", "pass": true }, { "id": "heading-order", "desc": "Exactly one h1, headings never skip a level, and there are at least 8 headings", "pass": true }, { "id": "landmarks", "desc": "Has header, nav, one main and footer landmarks, a lang attribute and a title", "pass": true }, { "id": "img-alt", "desc": "Every image has an alt attribute", "pass": true }, { "id": "form-labels", "desc": "The form has three fields (name, email, message), each with a visible