{ "native": { "attempts": [ { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-dispatch-spent-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef dispatch_iff_authorized(record, policy):\n if not policy:\n return None\n ", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 393162773, "process_cpu_ns": 768502967, "process_maxrss_kib": 523568 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 79, "total_tokens": 103 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-dispatch-spent-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"authorized\" }", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 303317234, "process_cpu_ns": 550647848, "process_maxrss_kib": 538160 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 79, "total_tokens": 87 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-dispatch-spent-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef dispatch_iff_authorized(record, policy):\n if not policy:\n return None\n if not record:\n return None\n if record.authorized:\n return record\n if record.nonce_spent:\n return None\n return record\n```", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 598228449, "process_cpu_ns": 1156957245, "process_maxrss_kib": 538288 }, "schemaValid": false, "tokens": { "completion_tokens": 58, "prompt_tokens": 79, "total_tokens": 137 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-dispatch-spent-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"authorized\" }", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 296933805, "process_cpu_ns": 544388280, "process_maxrss_kib": 538288 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 79, "total_tokens": 87 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-dispatch-spent-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Dispatch iff authorized is true and nonce_spent is false\",\n ", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 908653536, "process_cpu_ns": 1800341455, "process_maxrss_kib": 551056 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 79, "total_tokens": 103 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-dispatch-spent-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Dispatch iff authorized is true and nonce_spent is false\" }", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 1067655324, "process_cpu_ns": 1971111964, "process_maxrss_kib": 556816 }, "schemaValid": true, "tokens": { "completion_tokens": 23, "prompt_tokens": 79, "total_tokens": 102 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-dispatch-spent-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Dispatch iff authorized is true and nonce_spent is false\",\n \"record\": {\n \"authorized\": true,\n \"nonce_spent\": true\n }\n}", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 1319904734, "process_cpu_ns": 2602857253, "process_maxrss_kib": 556816 }, "schemaValid": false, "tokens": { "completion_tokens": 47, "prompt_tokens": 79, "total_tokens": 126 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-dispatch-spent-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Dispatch iff authorized is true and nonce_spent is false\" }", "pair": "policy-dispatch-spent", "pairRole": "b", "resources": { "elapsed_ns": 1063191871, "process_cpu_ns": 1962634586, "process_maxrss_kib": 557584 }, "schemaValid": true, "tokens": { "completion_tokens": 23, "prompt_tokens": 79, "total_tokens": 102 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-empty-list", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": [] }", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 229733252, "process_cpu_ns": 423945786, "process_maxrss_kib": 557584 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-empty-list", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": [\n {\n \"key\": \"value1\",\n \"value\": \"value2\"\n },\n {\n \"key\": \"value2\",\n \"value\": \"value3\"\n }\n ]\n}\n```", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 529781402, "process_cpu_ns": 1024814699, "process_maxrss_kib": 557584 }, "schemaValid": false, "tokens": { "completion_tokens": 55, "prompt_tokens": 64, "total_tokens": 119 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-empty-list", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": [] }", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 258479578, "process_cpu_ns": 473238365, "process_maxrss_kib": 557584 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-empty-list", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": []\n}", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 596017791, "process_cpu_ns": 1184883272, "process_maxrss_kib": 557584 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-empty-list", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": [ \"payload\" ] }", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 653118799, "process_cpu_ns": 1244211163, "process_maxrss_kib": 557584 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 64, "total_tokens": 74 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-empty-list", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": []\n}", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 566794679, "process_cpu_ns": 1126688110, "process_maxrss_kib": 557584 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-empty-list", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": [ \"payload\" ] }", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 653294400, "process_cpu_ns": 1244094342, "process_maxrss_kib": 557584 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 64, "total_tokens": 74 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-empty-list", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": [\n {\n \"key\": \"value1\",\n \"value\": \"", "pair": "typed-pair-6", "pairRole": "a", "resources": { "elapsed_ns": 311829281, "process_cpu_ns": 608561337, "process_maxrss_kib": 557584 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 64, "total_tokens": 88 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-publish-reader-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_evidence(reader, record, policy):\n \"\"\"Publish iff evidence_complete is true and the selected reader supports the evidence_profile.\n\n Args:\n reader (pandas.DataFrame): The reader for which to publish the evidence.\n record (pandas.DataFrame): The record for which to publish the evidence.\n policy (str): The policy for which to publish the evidence.\n\n Returns:\n dict: A dictionary containing the following keys", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 926536709, "process_cpu_ns": 1786709595, "process_maxrss_kib": 557584 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 98, "total_tokens": 194 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-publish-reader-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"v2\" }", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 365021557, "process_cpu_ns": 670223378, "process_maxrss_kib": 558480 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 98, "total_tokens": 107 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-publish-reader-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"hold\",\n \"record\": {\n \"evidence_complete\": true,\n \"evidence", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 1048445903, "process_cpu_ns": 2080508928, "process_maxrss_kib": 558608 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 98, "total_tokens": 122 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-publish-reader-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"hold\" }", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 892479876, "process_cpu_ns": 1734323296, "process_maxrss_kib": 558864 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 98, "total_tokens": 106 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-publish-reader-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"hold\",\n \"record\": {\n \"evidence_complete\": true,\n \"evidence_profile\": \"v2\",\n \"reader_supports\": [\"v1\", \"v2\"]\n }\n}", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 1500815323, "process_cpu_ns": 2963570595, "process_maxrss_kib": 558864 }, "schemaValid": false, "tokens": { "completion_tokens": 50, "prompt_tokens": 98, "total_tokens": 148 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-publish-reader-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"hold\" }", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 862766002, "process_cpu_ns": 1673906399, "process_maxrss_kib": 558864 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 98, "total_tokens": 106 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-publish-reader-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_evidence(reader, record, policy):\n \"\"\"Publish iff evidence_complete is", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 417680885, "process_cpu_ns": 819087645, "process_maxrss_kib": 558864 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 98, "total_tokens": 122 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-publish-reader-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"v2\" }", "pair": "policy-publish-reader", "pairRole": "a", "resources": { "elapsed_ns": 362274396, "process_cpu_ns": 666412500, "process_maxrss_kib": 558864 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 98, "total_tokens": 107 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-escaped-backslash", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\\b\" }", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 299063069, "process_cpu_ns": 529792951, "process_maxrss_kib": 558864 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-escaped-backslash", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"a\\\\b\"\n}", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 664561688, "process_cpu_ns": 1319588116, "process_maxrss_kib": 558864 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 67, "total_tokens": 79 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-escaped-backslash", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\\b\" }", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 760256447, "process_cpu_ns": 1454494611, "process_maxrss_kib": 558864 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-escaped-backslash", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"a\\\\b\"\n}", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 664430212, "process_cpu_ns": 1319453118, "process_maxrss_kib": 558864 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 67, "total_tokens": 79 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-escaped-backslash", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\\b\" }", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 688513945, "process_cpu_ns": 1311233701, "process_maxrss_kib": 558992 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-escaped-backslash", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"a\": {\n \"type\": \"string\",\n \"value", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 327833573, "process_cpu_ns": 640512490, "process_maxrss_kib": 558992 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 67, "total_tokens": 91 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-escaped-backslash", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\\b\" }", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 297417507, "process_cpu_ns": 528733320, "process_maxrss_kib": 558992 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-escaped-backslash", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"a\": {\n \"type\": \"string\",\n \"value\": \"a\\\\b\"\n }\n }\n}\n```", "pair": "typed-pair-5", "pairRole": "b", "resources": { "elapsed_ns": 428467010, "process_cpu_ns": 832180287, "process_maxrss_kib": 558992 }, "schemaValid": false, "tokens": { "completion_tokens": 38, "prompt_tokens": 67, "total_tokens": 105 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-empty-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 1111818173, "process_cpu_ns": 2209366011, "process_maxrss_kib": 561816 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 125, "total_tokens": 139 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-empty-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 1107633233, "process_cpu_ns": 2151404915, "process_maxrss_kib": 561816 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-empty-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 1105983714, "process_cpu_ns": 2199786435, "process_maxrss_kib": 561816 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 125, "total_tokens": 139 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-empty-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 1125492139, "process_cpu_ns": 2185943795, "process_maxrss_kib": 561816 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-empty-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record, policy):", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 484682840, "process_cpu_ns": 952256269, "process_maxrss_kib": 561816 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 125, "total_tokens": 149 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-empty-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"\" }", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 454745624, "process_cpu_ns": 845064230, "process_maxrss_kib": 561816 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-empty-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record, policy):\n if record.known:\n return record\n else:\n return record.json_object()\n```", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 651863634, "process_cpu_ns": 1270974134, "process_maxrss_kib": 561816 }, "schemaValid": false, "tokens": { "completion_tokens": 46, "prompt_tokens": 125, "total_tokens": 171 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-empty-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"\" }", "pair": "grounded-empty", "pairRole": "a", "resources": { "elapsed_ns": 453776354, "process_cpu_ns": 844332485, "process_maxrss_kib": 561944 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-admit-signature-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"reject\" }", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 795502776, "process_cpu_ns": 1532491604, "process_maxrss_kib": 563480 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-admit-signature-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"reject\",\n \"record\": {\n \"signature\": \"invalid\",\n \"now\": 10,\n \"expiry\": 11\n }\n}", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 1263160205, "process_cpu_ns": 2494569685, "process_maxrss_kib": 563480 }, "schemaValid": false, "tokens": { "completion_tokens": 41, "prompt_tokens": 85, "total_tokens": 126 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-admit-signature-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"reject\" }", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 789302626, "process_cpu_ns": 1520619720, "process_maxrss_kib": 563480 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-admit-signature-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef reject(record):\n record = record.replace(\"invalid\", \"not valid\")\n record =", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 371305555, "process_cpu_ns": 726578142, "process_maxrss_kib": 563480 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-admit-signature-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"invalid\" }", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 312417117, "process_cpu_ns": 574283449, "process_maxrss_kib": 563480 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-admit-signature-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef reject(record):\n record = record.replace(\"invalid\", \"not valid\")\n record = record.replace(\"now\", \"not now\")\n record = record.replace(\"expiry\", \"not expired\")\n record = record.replace(\"decision\", \"not decision\")\n record = record.replace(\"decision\", \"not decision\")\n record = record.replace(\"decision\", \"not decision\")\n record = record.replace(\"decision\",", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 894738613, "process_cpu_ns": 1720695215, "process_maxrss_kib": 563480 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 85, "total_tokens": 181 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-admit-signature-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"invalid\" }", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 316029823, "process_cpu_ns": 581173999, "process_maxrss_kib": 563480 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-admit-signature-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"reject\",\n \"record\": {\n \"signature\": \"invalid\",\n \"now", "pair": "policy-admit-signature", "pairRole": "b", "resources": { "elapsed_ns": 988226353, "process_cpu_ns": 1958909127, "process_maxrss_kib": 563480 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-string-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"city\": \"Lima\"\n}", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1075694135, "process_cpu_ns": 2141066633, "process_maxrss_kib": 563480 }, "schemaValid": false, "tokens": { "completion_tokens": 11, "prompt_tokens": 127, "total_tokens": 138 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-string-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1123367997, "process_cpu_ns": 2183798140, "process_maxrss_kib": 563480 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 127, "total_tokens": 138 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-string-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_record(record):\n if", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 480231647, "process_cpu_ns": 944394114, "process_maxrss_kib": 563480 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 127, "total_tokens": 151 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-string-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Lima\" }", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 496745766, "process_cpu_ns": 907978798, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 127, "total_tokens": 141 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-string-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_record(record):\n if record.city == \"Lima\":\n return {\n \"city\": record.city,\n \"value\": record.value\n }\n else:\n return record\n```", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 747542923, "process_cpu_ns": 1454521647, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 60, "prompt_tokens": 127, "total_tokens": 187 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-string-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Lima\" }", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 496109525, "process_cpu_ns": 906934137, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 127, "total_tokens": 141 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-string-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"city\": \"Lima\"\n}", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1070192960, "process_cpu_ns": 2129990304, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 11, "prompt_tokens": 127, "total_tokens": 138 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-string-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1121326901, "process_cpu_ns": 2178631235, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 127, "total_tokens": 138 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-null-value", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": null }", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 569121476, "process_cpu_ns": 1101549364, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-null-value", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": null\n}\n```", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 250239411, "process_cpu_ns": 491277725, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-null-value", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": null }", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 229321162, "process_cpu_ns": 423190752, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-null-value", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": null\n}\n```", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 244358125, "process_cpu_ns": 479677096, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-null-value", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": null }", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 230748453, "process_cpu_ns": 425218717, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-null-value", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": null\n}", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 566571046, "process_cpu_ns": 1126165869, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-null-value", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": null }", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 567617601, "process_cpu_ns": 1098807642, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-null-value", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": null\n}", "pair": "typed-pair-2", "pairRole": "a", "resources": { "elapsed_ns": 569391233, "process_cpu_ns": 1131822350, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-false-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if not record:\n return None\n if", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 465431375, "process_cpu_ns": 914571665, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-false-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 491382917, "process_cpu_ns": 891300182, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-false-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if not record:\n return None\n if not isinstance(record, dict):\n return None\n if not isinstance(record, (list, tuple)):\n return None\n if not isinstance(record, (str, int, float, bool, list, tuple)):\n return None\n if not isinstance(record, (list, tuple)):\n return None\n if not isinstance(record, (str", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 983125419, "process_cpu_ns": 1901198813, "process_maxrss_kib": 563608 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 121, "total_tokens": 217 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-false-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 490179517, "process_cpu_ns": 888764063, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-false-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 1067060208, "process_cpu_ns": 2122358491, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-false-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 1075176494, "process_cpu_ns": 2086893609, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-false-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 1076604246, "process_cpu_ns": 2140997563, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-false-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-false", "pairRole": "b", "resources": { "elapsed_ns": 1079605163, "process_cpu_ns": 2095369226, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-boolean-true", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": true }", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 229207304, "process_cpu_ns": 422686573, "process_maxrss_kib": 563608 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-boolean-true", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": true\n}\n```", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 245194235, "process_cpu_ns": 481248702, "process_maxrss_kib": 563736 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-boolean-true", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": true }", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 229488973, "process_cpu_ns": 423180955, "process_maxrss_kib": 563736 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-boolean-true", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": true\n}", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 564512007, "process_cpu_ns": 1122203935, "process_maxrss_kib": 563736 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-boolean-true", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": true }", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 568403206, "process_cpu_ns": 1099842794, "process_maxrss_kib": 563736 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-boolean-true", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": true\n}", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 568078275, "process_cpu_ns": 1129123472, "process_maxrss_kib": 563736 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-boolean-true", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": true }", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 569268641, "process_cpu_ns": 1101527528, "process_maxrss_kib": 563736 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-boolean-true", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": true\n}\n```", "pair": "typed-pair-1", "pairRole": "b", "resources": { "elapsed_ns": 245234743, "process_cpu_ns": 481251756, "process_maxrss_kib": 563736 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-matching-record-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record):\n if record.known:\n return record\n else:\n return record.get(\"subject\", None)\n```", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 684070712, "process_cpu_ns": 1337312106, "process_maxrss_kib": 563736 }, "schemaValid": false, "tokens": { "completion_tokens": 46, "prompt_tokens": 141, "total_tokens": 187 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-matching-record-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Omar\" }", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 534442290, "process_cpu_ns": 983788617, "process_maxrss_kib": 563864 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-matching-record-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 1224336903, "process_cpu_ns": 2437522148, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-matching-record-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 1223543071, "process_cpu_ns": 2383599175, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 141, "total_tokens": 152 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-matching-record-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 1216841663, "process_cpu_ns": 2421941757, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-matching-record-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 1225385442, "process_cpu_ns": 2387962007, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 141, "total_tokens": 152 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-matching-record-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record):\n if", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 522063707, "process_cpu_ns": 1027344667, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 141, "total_tokens": 165 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-matching-record-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Omar\" }", "pair": "grounded-matching-record", "pairRole": "b", "resources": { "elapsed_ns": 534585890, "process_cpu_ns": 985309714, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-false-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"null\" }", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 473462071, "process_cpu_ns": 868807420, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 124, "total_tokens": 137 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-false-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"enabled\": false }", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 961176997, "process_cpu_ns": 1915690413, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 6, "prompt_tokens": 124, "total_tokens": 130 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-false-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": false, \"value\": null }", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 1103443293, "process_cpu_ns": 2143898195, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 124, "total_tokens": 135 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-false-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"enabled\": false }", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 963702666, "process_cpu_ns": 1920652488, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 6, "prompt_tokens": 124, "total_tokens": 130 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-false-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": false, \"value\": null }", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 1100158061, "process_cpu_ns": 2137091402, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 124, "total_tokens": 135 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-false-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if record.enabled:\n return record\n ", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 471893036, "process_cpu_ns": 927656781, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 124, "total_tokens": 148 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-false-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"null\" }", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 474441594, "process_cpu_ns": 870910836, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 124, "total_tokens": 137 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-false-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if record.enabled:\n return record\n else:\n return record.value\n```", "pair": "grounded-false", "pairRole": "a", "resources": { "elapsed_ns": 542724024, "process_cpu_ns": 1063402649, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 33, "prompt_tokens": 124, "total_tokens": 157 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-unicode-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"M\u00e1laga\"\n}", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 683951603, "process_cpu_ns": 1356948578, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 68, "total_tokens": 81 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-unicode-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"M\u00e1laga\" }", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 715910500, "process_cpu_ns": 1358398549, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-unicode-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"M\u00e1laga\"\n}", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 679823921, "process_cpu_ns": 1349242818, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 68, "total_tokens": 81 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-unicode-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"M\u00e1laga\" }", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 748276006, "process_cpu_ns": 1417887176, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-unicode-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"M\u00e1laga\": {\n \"name\": \"M\u00e1", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 331359434, "process_cpu_ns": 646685420, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 68, "total_tokens": 92 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-unicode-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"M\u00e1laga\" }", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 315215277, "process_cpu_ns": 558018292, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-unicode-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"M\u00e1laga\": {\n \"name\": \"M\u00e1laga\",\n \"population\": 1000000\n }\n }\n}\n```", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 498807138, "process_cpu_ns": 966871632, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 47, "prompt_tokens": 68, "total_tokens": 115 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-unicode-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"M\u00e1laga\" }", "pair": "typed-pair-7", "pairRole": "a", "resources": { "elapsed_ns": 313573582, "process_cpu_ns": 553629959, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-admit-expiry-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"reject\" }", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 790697676, "process_cpu_ns": 1522998571, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-admit-expiry-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"reject\",\n \"record\": {\n \"signature\": \"valid\",\n \"now\": 10,\n \"expiry\": 11\n }\n}", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 1251701690, "process_cpu_ns": 2472283906, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 41, "prompt_tokens": 85, "total_tokens": 126 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-admit-expiry-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"reject\" }", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 789262550, "process_cpu_ns": 1519581776, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-admit-expiry-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if record.signature == \"valid\":\n ", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 371853062, "process_cpu_ns": 727498671, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-admit-expiry-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"valid\" }", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 319158304, "process_cpu_ns": 588248101, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-admit-expiry-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if record.signature == \"valid\":\n return record\n elif record.now == 10:\n return record\n else:\n return record\n```", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 546559871, "process_cpu_ns": 1060471078, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 47, "prompt_tokens": 85, "total_tokens": 132 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-admit-expiry-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"valid\" }", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 316795585, "process_cpu_ns": 581248649, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-admit-expiry-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"reject\",\n \"record\": {\n \"signature\": \"valid\",\n \"now", "pair": "policy-admit-expiry", "pairRole": "a", "resources": { "elapsed_ns": 960182197, "process_cpu_ns": 1902729392, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "long96", "correct": true, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-null-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 1105529662, "process_cpu_ns": 2198701125, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 125, "total_tokens": 139 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-null-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 1117426120, "process_cpu_ns": 2170119568, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-null-a", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record,", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 487720654, "process_cpu_ns": 958424001, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 125, "total_tokens": 149 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-null-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"null\" }", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 480874605, "process_cpu_ns": 881738003, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 125, "total_tokens": 138 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-null-a", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record, policy):\n \"\"\"\n Returns a JSON object with keys known and value.\n \"\"\"\n if record.known:\n return record\n elif record.known == True:\n return record\n elif record.known == False:\n return record\n else:\n return record\n```", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 900404206, "process_cpu_ns": 1743241119, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 80, "prompt_tokens": 125, "total_tokens": 205 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-null-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"null\" }", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 481503647, "process_cpu_ns": 883798567, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 125, "total_tokens": 138 } }, { "configuration": "short24", "correct": true, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-null-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 1111097411, "process_cpu_ns": 2209542555, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 125, "total_tokens": 139 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-null-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-null", "pairRole": "a", "resources": { "elapsed_ns": 1120800870, "process_cpu_ns": 2178078901, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-publish-consent-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 681506559, "process_cpu_ns": 1304861192, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-publish-consent-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n if record[\"checks\"] and record[\"consent", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 328759627, "process_cpu_ns": 642824395, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-publish-consent-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 296445161, "process_cpu_ns": 533355932, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-publish-consent-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n if record[\"checks\"] and record[\"consent\"]:\n return {\"decision\": record[\"decision\"]}\n```", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 428252578, "process_cpu_ns": 832601336, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 37, "prompt_tokens": 70, "total_tokens": 107 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-publish-consent-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 300738286, "process_cpu_ns": 539343669, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-publish-consent-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": true,\n \"consent", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 849814111, "process_cpu_ns": 1682758443, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-publish-consent-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 715494368, "process_cpu_ns": 1371475392, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-publish-consent-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": true,\n \"consent\": false\n }\n}", "pair": "policy-publish-consent", "pairRole": "b", "resources": { "elapsed_ns": 965064446, "process_cpu_ns": 1907537520, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 30, "prompt_tokens": 70, "total_tokens": 100 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-string-number", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"name\": \"007\",\n \"description\": \"A", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 323543271, "process_cpu_ns": 631621539, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 67, "total_tokens": 91 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-string-number", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"007\" }", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 298634378, "process_cpu_ns": 530315040, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-string-number", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"name\": \"007\",\n \"description\": \"A JSON object with only key-value pairs.\",\n \"data\": [\n {\n \"key\": \"value1\",\n \"value\": \"value1\",\n \"value2\": \"value2\",\n \"value3\": \"value3\",\n \"value4\": \"value4\",\n \"value5\": \"value5\",\n \"value6", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 840046815, "process_cpu_ns": 1612985965, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 67, "total_tokens": 163 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-string-number", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"007\" }", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 298314838, "process_cpu_ns": 530119221, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-string-number", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"007\"\n}", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 677005697, "process_cpu_ns": 1344050566, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 67, "total_tokens": 79 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-string-number", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"007\" }", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 712919210, "process_cpu_ns": 1359828358, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-string-number", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"007\"\n}", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 660328887, "process_cpu_ns": 1311071015, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 67, "total_tokens": 79 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-string-number", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"007\" }", "pair": "typed-pair-3", "pairRole": "a", "resources": { "elapsed_ns": 710268471, "process_cpu_ns": 1343926784, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-recover-committed-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 333340355, "process_cpu_ns": 615963837, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-recover-committed-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry(record, policy):\n if retry_authorized:\n return record\n else:\n return record\n```", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 436752369, "process_cpu_ns": 853124415, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 27, "prompt_tokens": 94, "total_tokens": 121 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-recover-committed-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 360560368, "process_cpu_ns": 668611251, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--policy-recover-committed-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"execution\": \"error\",\n \"committed\": true,\n \"retry_authorized\": true\n}", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 1022411649, "process_cpu_ns": 2026304489, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-recover-committed-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"commit\" }", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 829458194, "process_cpu_ns": 1607213635, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-recover-committed-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"execution\": \"error\",\n \"committed\": true,\n \"retry_authorized\": true\n}", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 1036068688, "process_cpu_ns": 2054088512, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-recover-committed-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"commit\" }", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 828472472, "process_cpu_ns": 1605298594, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-recover-committed-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry(record, policy):\n if retry_authorized:\n return record\n else:\n return", "pair": "policy-recover-committed", "pairRole": "b", "resources": { "elapsed_ns": 396736793, "process_cpu_ns": 776903967, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-integer-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(records, field_name):\n if field_name in records:\n return records[field_name]\n else:\n return records[field_name]\n```", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 723978908, "process_cpu_ns": 1411172270, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 55, "prompt_tokens": 131, "total_tokens": 186 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-integer-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"2\" }", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 498478387, "process_cpu_ns": 920306180, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 131, "total_tokens": 144 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-integer-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"rooms\": 2\n}", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1086371753, "process_cpu_ns": 2164621062, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 131, "total_tokens": 140 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-integer-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1171274476, "process_cpu_ns": 2279363856, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 131, "total_tokens": 142 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-integer-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"rooms\": 2\n}", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1074617556, "process_cpu_ns": 2140387524, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 131, "total_tokens": 140 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-integer-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1159056906, "process_cpu_ns": 2255469479, "process_maxrss_kib": 564120 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 131, "total_tokens": 142 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-integer-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(records, field_", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 501018669, "process_cpu_ns": 985431853, "process_maxrss_kib": 564120 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 131, "total_tokens": 155 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-integer-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"2\" }", "pair": "grounded-integer-conflict", "pairRole": "b", "resources": { "elapsed_ns": 499478100, "process_cpu_ns": 920308362, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 131, "total_tokens": 144 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-integer-seven", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": 7 }", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 249064301, "process_cpu_ns": 455790181, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-integer-seven", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": 7\n}", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 604017373, "process_cpu_ns": 1200041439, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-integer-seven", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": 7 }", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 613043252, "process_cpu_ns": 1184206371, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-integer-seven", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": 7\n}", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 598031780, "process_cpu_ns": 1188553396, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-integer-seven", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": 7 }", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 606188389, "process_cpu_ns": 1169364120, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-integer-seven", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": 7\n}\n```", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 257885809, "process_cpu_ns": 505865451, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 14, "prompt_tokens": 65, "total_tokens": 79 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-integer-seven", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": 7 }", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 246649095, "process_cpu_ns": 452158920, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-integer-seven", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": 7\n}\n```", "pair": "typed-pair-3", "pairRole": "b", "resources": { "elapsed_ns": 257794027, "process_cpu_ns": 505785549, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 14, "prompt_tokens": 65, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-publish-reader-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"hold\",\n \"record\": {\n \"evidence_complete\": true,\n \"evidence", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 1016032302, "process_cpu_ns": 2014861669, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-publish-reader-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"hold\" }", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 821872806, "process_cpu_ns": 1592303599, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-publish-reader-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"hold\",\n \"record\": {\n \"evidence_complete\": true,\n \"evidence_profile\": \"v2\",\n \"reader_supports\": [\"v1\"]\n }\n}", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 1384818424, "process_cpu_ns": 2735646533, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 46, "prompt_tokens": 94, "total_tokens": 140 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-publish-reader-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"hold\" }", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 829043414, "process_cpu_ns": 1606415990, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-publish-reader-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef publish_evidence(reader, record, policy):", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 396560356, "process_cpu_ns": 776553026, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-publish-reader-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"v2\" }", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 353383614, "process_cpu_ns": 647699594, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 94, "total_tokens": 103 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-publish-reader-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef publish_evidence(reader, record, policy):\n if reader.is_v1:\n return {\"evidence_complete\": True, \"evidence_profile\": \"v2\"}\n else:\n return {\"evidence_complete\": True, \"evidence_profile\": \"v2\"}\n```", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 757681049, "process_cpu_ns": 1465663540, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 74, "prompt_tokens": 94, "total_tokens": 168 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-publish-reader-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"v2\" }", "pair": "policy-publish-reader", "pairRole": "b", "resources": { "elapsed_ns": 351214587, "process_cpu_ns": 642934479, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 94, "total_tokens": 103 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-boolean-false", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": false }", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 571207630, "process_cpu_ns": 1105939385, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-boolean-false", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": false\n}", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 574918983, "process_cpu_ns": 1142435371, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-boolean-false", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": false }", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 576318171, "process_cpu_ns": 1115807133, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-boolean-false", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": false\n}\n```", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 254186517, "process_cpu_ns": 499010321, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-boolean-false", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": false }", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 232965629, "process_cpu_ns": 428432995, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-boolean-false", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": false\n}\n```", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 247238960, "process_cpu_ns": 485021432, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-boolean-false", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": false }", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 229685615, "process_cpu_ns": 423640245, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-boolean-false", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": false\n}", "pair": "typed-pair-1", "pairRole": "a", "resources": { "elapsed_ns": 569236032, "process_cpu_ns": 1131498542, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-zero-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 1081233408, "process_cpu_ns": 2150914900, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-zero-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 1085023773, "process_cpu_ns": 2105683854, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-zero-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record):\n if", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 466921995, "process_cpu_ns": 917182810, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-zero-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 497492786, "process_cpu_ns": 899817141, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-zero-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record):\n if record.age == 0:\n return {\n \"value\": record.age,\n \"known\": True,\n \"known_value\": record.age,\n }\n else:\n return {\n \"value\": record.age,\n \"known\": False,\n \"known_value\": record.age,\n }\n```", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 970172841, "process_cpu_ns": 1875952299, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 94, "prompt_tokens": 121, "total_tokens": 215 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-zero-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 491932127, "process_cpu_ns": 890992762, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-zero-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 1064035422, "process_cpu_ns": 2116793943, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-zero-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-zero", "pairRole": "b", "resources": { "elapsed_ns": 1076486892, "process_cpu_ns": 2089101514, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-ordered-list", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": [\"second\", \"first\"] }", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 721561747, "process_cpu_ns": 1370305609, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 69, "total_tokens": 80 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-ordered-list", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": [\n {\n \"key\": \"second\",\n \"value\": \"first", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 330960049, "process_cpu_ns": 646623924, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 69, "total_tokens": 93 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-ordered-list", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": [ \"second\", \"first\" ] }", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 337766034, "process_cpu_ns": 593665309, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 69, "total_tokens": 82 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-ordered-list", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": [\n {\n \"key\": \"second\",\n \"value\": \"first\"\n }\n ]\n}\n```", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 396116443, "process_cpu_ns": 771069612, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 33, "prompt_tokens": 69, "total_tokens": 102 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-ordered-list", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": [ \"second\", \"first\" ] }", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 333435765, "process_cpu_ns": 586202357, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 69, "total_tokens": 82 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-ordered-list", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": [\"second\", \"first\"]\n}", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 688416527, "process_cpu_ns": 1366416970, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 69, "total_tokens": 82 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-ordered-list", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": [\"second\", \"first\"] }", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 722908807, "process_cpu_ns": 1372749298, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 69, "total_tokens": 80 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-ordered-list", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": [\"second\", \"first\"]\n}", "pair": "typed-pair-6", "pairRole": "b", "resources": { "elapsed_ns": 692406808, "process_cpu_ns": 1374315796, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 69, "total_tokens": 82 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-admit-signature-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if record.signature == \"valid\":\n ", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 373537705, "process_cpu_ns": 730460159, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-admit-signature-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"valid\" }", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 314428534, "process_cpu_ns": 577456619, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-admit-signature-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef get_json_object(record, policy):\n if record.signature == \"valid\":\n return record\n elif record.now == 10:\n return record\n else:\n return record\n```", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 539748683, "process_cpu_ns": 1048657887, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 47, "prompt_tokens": 85, "total_tokens": 132 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-admit-signature-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"valid\" }", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 315335050, "process_cpu_ns": 579635732, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-admit-signature-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"reject\",\n \"record\": {\n \"signature\": \"valid\",\n \"now", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 961486133, "process_cpu_ns": 1905874520, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-admit-signature-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"reject\" }", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 790247922, "process_cpu_ns": 1521477652, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-admit-signature-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"reject\",\n \"record\": {\n \"signature\": \"valid\",\n \"now\": 10,\n \"expiry\": 11\n }\n}", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 1246146699, "process_cpu_ns": 2462994912, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 41, "prompt_tokens": 85, "total_tokens": 126 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-admit-signature-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"reject\" }", "pair": "policy-admit-signature", "pairRole": "a", "resources": { "elapsed_ns": 788419650, "process_cpu_ns": 1518622478, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-dispatch-spent-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"authorized\" }", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 298306057, "process_cpu_ns": 546570098, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 79, "total_tokens": 87 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-dispatch-spent-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef dispatch_iff(authorized, nonce_spent):\n record = {\n \"authorized\": True,\n \"nonce_spent\": False\n }\n if authorized:\n record[\"authorized\"] = True\n else:\n record[\"nonce_spent\"] = False\n return record\n```", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 688861721, "process_cpu_ns": 1330536785, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 68, "prompt_tokens": 79, "total_tokens": 147 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-dispatch-spent-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"authorized\" }", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 303440066, "process_cpu_ns": 555705431, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 79, "total_tokens": 87 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-dispatch-spent-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Dispatch iff authorized is true and nonce_spent is false\",\n ", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 928092188, "process_cpu_ns": 1838315646, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 79, "total_tokens": 103 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-dispatch-spent-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"hold\" }", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 746057080, "process_cpu_ns": 1440352601, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 79, "total_tokens": 87 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-dispatch-spent-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Dispatch iff authorized is true and nonce_spent is false\",\n \"record\": {\n \"authorized\": true,\n \"nonce_spent\": false\n }\n}", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 1312905079, "process_cpu_ns": 2592285519, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 47, "prompt_tokens": 79, "total_tokens": 126 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-dispatch-spent-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"hold\" }", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 727192762, "process_cpu_ns": 1402146885, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 79, "total_tokens": 87 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-dispatch-spent-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef dispatch_iff(authorized, nonce_spent):\n record = {\n \"authorized", "pair": "policy-dispatch-spent", "pairRole": "a", "resources": { "elapsed_ns": 356035606, "process_cpu_ns": 696646480, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 79, "total_tokens": 103 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-recover-authorized-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry_authorized(record, policy):\n if retry_authorized:\n return {\"decision\": \"authorized\"}\n else:\n return {\"decision\": \"error\"}\n```\n\nThis function checks if the retry_authorized flag is set to `True` and returns a JSON object with the string \"authorized\". If the retry_authorized flag is set to `False` and the retry_authorized flag is not set, the retry_authorized flag is set", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 889530553, "process_cpu_ns": 1712693396, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 85, "total_tokens": 181 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-recover-authorized-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 314084532, "process_cpu_ns": 576782512, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-recover-authorized-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"decision\",\n \"execution\": \"error\",\n \"committed\": false,\n \"", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 1046314948, "process_cpu_ns": 2072242954, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-recover-authorized-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"decision\" }", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 793440252, "process_cpu_ns": 1531561799, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-recover-authorized-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"decision\",\n \"execution\": \"error\",\n \"committed\": false,\n \"retry_authorized\": true\n}", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 1080612952, "process_cpu_ns": 2139584543, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 31, "prompt_tokens": 85, "total_tokens": 116 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-recover-authorized-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"decision\" }", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 763986529, "process_cpu_ns": 1476122360, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-recover-authorized-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry_authorized(record, policy):\n if retry_authorized:\n return {\"decision\": \"", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 369617552, "process_cpu_ns": 723698109, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-recover-authorized-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-authorized", "pairRole": "a", "resources": { "elapsed_ns": 314290155, "process_cpu_ns": 578153179, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-integer-zero", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": 0 }", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 247898650, "process_cpu_ns": 453907844, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-integer-zero", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": 0\n}", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 601682984, "process_cpu_ns": 1195937761, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-integer-zero", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": 0 }", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 606534408, "process_cpu_ns": 1170453733, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-integer-zero", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": 0\n}", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 597272502, "process_cpu_ns": 1187024929, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-integer-zero", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": 0 }", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 604775297, "process_cpu_ns": 1167048114, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-integer-zero", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": 0\n}\n```", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 257787266, "process_cpu_ns": 505918847, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 14, "prompt_tokens": 65, "total_tokens": 79 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-integer-zero", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": 0 }", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 246859604, "process_cpu_ns": 452664672, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-integer-zero", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": 0\n}\n```", "pair": "typed-pair-0", "pairRole": "a", "resources": { "elapsed_ns": 258024954, "process_cpu_ns": 506419487, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 14, "prompt_tokens": 65, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-integer-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"rooms\": 2\n}", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1025977308, "process_cpu_ns": 2043164269, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 125, "total_tokens": 134 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-integer-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1167477895, "process_cpu_ns": 2269260072, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-integer-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"rooms\": 2\n}", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1054065894, "process_cpu_ns": 2099353359, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 125, "total_tokens": 134 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-integer-conflict-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 1115597161, "process_cpu_ns": 2166604614, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-integer-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(records, field_", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 492604812, "process_cpu_ns": 968061025, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 125, "total_tokens": 149 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-integer-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"2\" }", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 517367218, "process_cpu_ns": 949591981, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 125, "total_tokens": 138 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-integer-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(records, field_name):\n if field_name in records:\n return records[field_name]\n else:\n return records[field_name]\n```", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 721963171, "process_cpu_ns": 1405540053, "process_maxrss_kib": 564248 }, "schemaValid": false, "tokens": { "completion_tokens": 55, "prompt_tokens": 125, "total_tokens": 180 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-integer-conflict-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"2\" }", "pair": "grounded-integer-conflict", "pairRole": "a", "resources": { "elapsed_ns": 482665652, "process_cpu_ns": 885409022, "process_maxrss_kib": 564248 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 125, "total_tokens": 138 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-string-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1181863317, "process_cpu_ns": 2299153630, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 134, "total_tokens": 145 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-string-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1172158348, "process_cpu_ns": 2332454139, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 134, "total_tokens": 148 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-string-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1185479053, "process_cpu_ns": 2305546151, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 134, "total_tokens": 145 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-string-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_record(record):\n if", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 521886921, "process_cpu_ns": 1026014714, "process_maxrss_kib": 566296 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 134, "total_tokens": 158 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-string-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Oslo\" }", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 524977847, "process_cpu_ns": 963029094, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 134, "total_tokens": 148 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-string-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_record(record):\n if record.city == \"Lima\":\n return {\n \"city\": record.city,\n \"value\": record.value\n }\n elif record.city == \"Oslo\":\n return {\n \"city\": record.city,\n \"value\": record.value\n }\n else:\n return None\n```", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1032952041, "process_cpu_ns": 1998790609, "process_maxrss_kib": 566296 }, "schemaValid": false, "tokens": { "completion_tokens": 90, "prompt_tokens": 134, "total_tokens": 224 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-string-conflict-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Oslo\" }", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 526388972, "process_cpu_ns": 965149798, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 134, "total_tokens": 148 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-string-conflict-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-string-conflict", "pairRole": "b", "resources": { "elapsed_ns": 1175596966, "process_cpu_ns": 2339835304, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 134, "total_tokens": 148 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-integer-negative", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": -7\n}", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 603632871, "process_cpu_ns": 1199032664, "process_maxrss_kib": 566296 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-integer-negative", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": -7 }", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 609268114, "process_cpu_ns": 1175032560, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-integer-negative", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": -7\n}\n```", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 262646976, "process_cpu_ns": 515068007, "process_maxrss_kib": 566296 }, "schemaValid": false, "tokens": { "completion_tokens": 14, "prompt_tokens": 65, "total_tokens": 79 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-integer-negative", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": -7 }", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 254264724, "process_cpu_ns": 464795731, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-integer-negative", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": -7\n}\n```", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 263790136, "process_cpu_ns": 517372805, "process_maxrss_kib": 566296 }, "schemaValid": false, "tokens": { "completion_tokens": 14, "prompt_tokens": 65, "total_tokens": 79 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-integer-negative", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": -7 }", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 254418063, "process_cpu_ns": 464826751, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-integer-negative", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": -7\n}", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 609491706, "process_cpu_ns": 1210702350, "process_maxrss_kib": 566296 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-integer-negative", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": -7 }", "pair": "typed-pair-0", "pairRole": "b", "resources": { "elapsed_ns": 613988697, "process_cpu_ns": 1185791826, "process_maxrss_kib": 566296 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-publish-checks-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 692645673, "process_cpu_ns": 1326354462, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-publish-checks-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n if record.checks and record.consent:", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 341922095, "process_cpu_ns": 667697546, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-publish-checks-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 304279468, "process_cpu_ns": 547760875, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-publish-checks-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n if record.checks and record.consent:\n return {\"decision\": record.decision}\n```", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 428838037, "process_cpu_ns": 833958484, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 35, "prompt_tokens": 70, "total_tokens": 105 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-publish-checks-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 302651565, "process_cpu_ns": 544280639, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-publish-checks-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": false,\n \"consent", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 875943043, "process_cpu_ns": 1734267349, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-publish-checks-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 695886364, "process_cpu_ns": 1332957058, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-publish-checks-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": false,\n \"consent\": true\n }\n}", "pair": "policy-publish-checks", "pairRole": "b", "resources": { "elapsed_ns": 1003454301, "process_cpu_ns": 1983140309, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 30, "prompt_tokens": 70, "total_tokens": 100 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-newline-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"top\": \"top\\nbottom\",\n \"top\": \"", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 340406682, "process_cpu_ns": 663857204, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 68, "total_tokens": 92 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-newline-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"top\\nbottom\" }", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 317369605, "process_cpu_ns": 562926565, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-newline-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\nbottom\",\n \"top\": \"top\\", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 879604546, "process_cpu_ns": 1690022179, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 68, "total_tokens": 164 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-newline-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"top\\nbottom\" }", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 317482247, "process_cpu_ns": 562369489, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-newline-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"top\\nbottom\"\n}", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 688656357, "process_cpu_ns": 1366835589, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 68, "total_tokens": 81 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-newline-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"top\\nbottom\" }", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 719080217, "process_cpu_ns": 1365304603, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-newline-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"top\\nbottom\"\n}", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 688186646, "process_cpu_ns": 1366089108, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 68, "total_tokens": 81 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-newline-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"top\\nbottom\" }", "pair": "typed-pair-7", "pairRole": "b", "resources": { "elapsed_ns": 719345488, "process_cpu_ns": 1366852735, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 68, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-string-number-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 505747095, "process_cpu_ns": 914174432, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-string-number-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record, policy):\n if not record:\n return None\n if not isinstance(record, dict):\n return None\n if not isinstance(record, list):\n return None\n if not isinstance(record, (list, tuple)):\n return None\n if not isinstance(record, (str, int, float, bool, list, tuple)):\n return None\n if", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 1017118350, "process_cpu_ns": 1969077976, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 121, "total_tokens": 217 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-string-number-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 504366297, "process_cpu_ns": 914085233, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-string-number-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 1078369136, "process_cpu_ns": 2145028782, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-string-number-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 1087001324, "process_cpu_ns": 2109594642, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-string-number-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 1078229966, "process_cpu_ns": 2144472416, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-string-number-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 1100658935, "process_cpu_ns": 2136276624, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-string-number-b", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record, policy):", "pair": "grounded-string-number", "pairRole": "b", "resources": { "elapsed_ns": 476714378, "process_cpu_ns": 937224688, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-string-null", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"name\": \"null\",\n \"value\": null\n }\n}\n```", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 352103811, "process_cpu_ns": 686885324, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 27, "prompt_tokens": 65, "total_tokens": 92 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-string-null", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"null\" }", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 266421396, "process_cpu_ns": 480998873, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 65, "total_tokens": 73 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-string-null", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": null\n}", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 593229204, "process_cpu_ns": 1178847611, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 65, "total_tokens": 73 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-string-null", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": null }", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 592008478, "process_cpu_ns": 1146847750, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 65, "total_tokens": 71 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-string-null", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": null\n}", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 591897561, "process_cpu_ns": 1176481757, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 65, "total_tokens": 73 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-string-null", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": null }", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 589328356, "process_cpu_ns": 1141354214, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 65, "total_tokens": 71 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-string-null", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"name\": \"null\",\n \"value\": null\n }\n", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 326841043, "process_cpu_ns": 638360236, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 65, "total_tokens": 89 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-string-null", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"null\" }", "pair": "typed-pair-2", "pairRole": "b", "resources": { "elapsed_ns": 267507500, "process_cpu_ns": 483002278, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 65, "total_tokens": 73 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-empty-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"\" }", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 236209845, "process_cpu_ns": 434943433, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-empty-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"\"\n}", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 573606090, "process_cpu_ns": 1140103128, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-empty-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"payload\" }", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 622239427, "process_cpu_ns": 1191930404, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-empty-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"\"\n}", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 573495602, "process_cpu_ns": 1139958959, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-empty-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"payload\" }", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 618630886, "process_cpu_ns": 1185715522, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 64, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-empty-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": \"\"\n}\n```", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 250175094, "process_cpu_ns": 490867987, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-empty-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"\" }", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 235502173, "process_cpu_ns": 433052933, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 6, "prompt_tokens": 64, "total_tokens": 70 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-empty-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": \"\"\n}\n```", "pair": "typed-pair-4", "pairRole": "a", "resources": { "elapsed_ns": 251864563, "process_cpu_ns": 493682001, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 13, "prompt_tokens": 64, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-admit-expiry-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"valid\",\n \"decision_date\": 11,\n \"expiry_date", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 964154606, "process_cpu_ns": 1911486507, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-admit-expiry-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"admit\" }", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 794171529, "process_cpu_ns": 1527944841, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-admit-expiry-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"valid\",\n \"decision_date\": 11,\n \"expiry_date\": 11\n}", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 1076203222, "process_cpu_ns": 2131089655, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 30, "prompt_tokens": 85, "total_tokens": 115 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-admit-expiry-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"admit\" }", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 794790304, "process_cpu_ns": 1529617788, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 85, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-admit-expiry-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef reject(record):\n record.signature = \"valid\"\n record.now = 11", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 377885255, "process_cpu_ns": 739360810, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-admit-expiry-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"valid\" }", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 314675631, "process_cpu_ns": 578289645, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-admit-expiry-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef reject(record):\n record.signature = \"valid\"\n record.now = 11\n record.expiry = 11\n return record\n```", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 483832167, "process_cpu_ns": 942272164, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 38, "prompt_tokens": 85, "total_tokens": 123 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-admit-expiry-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"valid\" }", "pair": "policy-admit-expiry", "pairRole": "b", "resources": { "elapsed_ns": 317955491, "process_cpu_ns": 583035063, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-recover-authorized-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"decision\" }", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 776293233, "process_cpu_ns": 1500271404, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-recover-authorized-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"decision\",\n \"execution\": \"error\",\n \"committed\": false,\n \"retry_authorized\": false\n}", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 1112588938, "process_cpu_ns": 2199973991, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 31, "prompt_tokens": 85, "total_tokens": 116 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-recover-authorized-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"decision\" }", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 772017799, "process_cpu_ns": 1492304988, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-recover-authorized-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry_authorized(record, policy):\n \"\"\"\n Retrieves a JSON object with a string", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 379389916, "process_cpu_ns": 742400695, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-recover-authorized-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 316382282, "process_cpu_ns": 582056549, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-recover-authorized-b", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry_authorized(record, policy):\n \"\"\"\n Retrieves a JSON object with a string key decision.\n \"\"\"\n record = record.replace(\"error\", \"Failed to retry\").replace(\"committed\", \"False\").replace(\"retry_authorized\", \"False\").replace(\"decision\", \"\".join(\n [\"success\", \"success\", \"success\", \"success\", \"success\", \"success\", \"success\", \"success\", \"success", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 899568801, "process_cpu_ns": 1733003691, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 85, "total_tokens": 181 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-recover-authorized-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 312473649, "process_cpu_ns": 575051476, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 85, "total_tokens": 93 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-recover-authorized-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"decision\",\n \"execution\": \"error\",\n \"committed\": false,\n \"", "pair": "policy-recover-authorized", "pairRole": "b", "resources": { "elapsed_ns": 994289543, "process_cpu_ns": 1970602562, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 85, "total_tokens": 109 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-empty-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 1075250380, "process_cpu_ns": 2138399926, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-empty-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 1082833369, "process_cpu_ns": 2101408526, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-empty-b", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record,", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 472606563, "process_cpu_ns": 928786614, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-empty-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 498542819, "process_cpu_ns": 903384642, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-empty-b", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record, policy):\n \"\"\"\n Returns a JSON object with keys known and value.\n \"\"\"\n if not record:\n return None\n\n if not record.keys():\n return None\n\n if not record.values():\n return None\n\n if not record.keys().issubset(record.values()):\n return None\n\n return record.keys() + [record", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 1000708503, "process_cpu_ns": 1935115837, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 121, "total_tokens": 217 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-empty-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"123\" }", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 498074126, "process_cpu_ns": 901995358, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 121, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-empty-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 1079445833, "process_cpu_ns": 2146381981, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 121, "total_tokens": 135 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-empty-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-empty", "pairRole": "b", "resources": { "elapsed_ns": 1118312153, "process_cpu_ns": 2163410254, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-escaped-quote", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\"b\" }", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 694233024, "process_cpu_ns": 1322263034, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-escaped-quote", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"a\": {\n \"b\": \"value\"\n }\n ", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 329566660, "process_cpu_ns": 643623528, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 67, "total_tokens": 91 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-escaped-quote", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\"b\" }", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 300221330, "process_cpu_ns": 535660004, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-escaped-quote", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"a\": {\n \"b\": \"value\"\n }\n }\n}\n```", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 372360660, "process_cpu_ns": 725824018, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 29, "prompt_tokens": 67, "total_tokens": 96 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-escaped-quote", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\"b\" }", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 298656908, "process_cpu_ns": 532000196, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-escaped-quote", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"a\\\"b\"\n}", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 669204196, "process_cpu_ns": 1328980539, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 67, "total_tokens": 79 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-escaped-quote", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \"a\\\"b\" }", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 701740158, "process_cpu_ns": 1336940996, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 67, "total_tokens": 77 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-escaped-quote", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \"a\\\"b\"\n}", "pair": "typed-pair-5", "pairRole": "a", "resources": { "elapsed_ns": 668042441, "process_cpu_ns": 1326222185, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 67, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-recover-committed-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry(record, policy):\n if retry_authorized:\n return record\n else:\n return", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 396589803, "process_cpu_ns": 777639558, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-recover-committed-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 337518961, "process_cpu_ns": 623761074, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-recover-committed-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef retry(record, policy):\n if retry_authorized:\n return record\n else:\n return record.get(\"decision\", None)\n```", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 477781223, "process_cpu_ns": 932678283, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 34, "prompt_tokens": 94, "total_tokens": 128 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-recover-committed-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"success\" }", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 340558590, "process_cpu_ns": 628294036, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--policy-recover-committed-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"execution\": \"error\",\n \"committed\": false,\n \"retry_authorized\": true\n}", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 1022066022, "process_cpu_ns": 2026674196, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-recover-committed-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"commit\" }", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 857870373, "process_cpu_ns": 1664244342, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-recover-committed-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"execution\": \"error\",\n \"committed\": false,\n \"retry_authorized\": true\n}", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 1046319284, "process_cpu_ns": 2074060027, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 94, "total_tokens": 118 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-recover-committed-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"commit\" }", "pair": "policy-recover-committed", "pairRole": "a", "resources": { "elapsed_ns": 832898852, "process_cpu_ns": 1613568561, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 8, "prompt_tokens": 94, "total_tokens": 102 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-null-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"true\" }", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 471293585, "process_cpu_ns": 863524690, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 121, "total_tokens": 134 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-null-b", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record, policy):\n \"\"\"\n Returns a JSON object with keys known and value.\n \"\"\"\n if not record:\n return None\n\n if not isinstance(record, dict):\n return None\n\n if not isinstance(record, (list, tuple)):\n return None\n\n if not isinstance(record, (str, int, float, bool, list, tuple)):", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 1009318450, "process_cpu_ns": 1952015031, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 121, "total_tokens": 217 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-null-b", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"true\" }", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 467418054, "process_cpu_ns": 857034266, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 121, "total_tokens": 134 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-null-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"reviewer\": \"John Doe\",\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 1234021278, "process_cpu_ns": 2449952392, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-null-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 1086839698, "process_cpu_ns": 2109039334, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-null-b", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"reviewer\": \"John Doe\",\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 1244893525, "process_cpu_ns": 2471432053, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-null-b", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 1087604319, "process_cpu_ns": 2110733524, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 121, "total_tokens": 132 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-null-b", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record,", "pair": "grounded-null", "pairRole": "b", "resources": { "elapsed_ns": 472686021, "process_cpu_ns": 929603567, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 121, "total_tokens": 145 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-string-number-a", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record, policy):\n \"\"\"\n Returns a JSON object with keys known and value.\n \"\"\"\n if record.code == \"007\":\n return {\n \"code\": \"007\",\n \"value\": record.value\n }\n elif record.code == \"008\":\n return {\n \"code\": \"008\",", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 1031465197, "process_cpu_ns": 1996808871, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 128, "total_tokens": 224 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-string-number-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"007\" }", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 523726527, "process_cpu_ns": 952543020, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 128, "total_tokens": 143 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-string-number-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"code\": \"007\"\n}", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 1105901764, "process_cpu_ns": 2201061970, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 128, "total_tokens": 140 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-string-number-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 1142909948, "process_cpu_ns": 2222357834, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 128, "total_tokens": 139 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-string-number-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"code\": \"007\"\n}", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 1096691885, "process_cpu_ns": 2182600135, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 12, "prompt_tokens": 128, "total_tokens": 140 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-string-number-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 1134475550, "process_cpu_ns": 2204674594, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 128, "total_tokens": 139 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-string-number-a", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record,", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 485425321, "process_cpu_ns": 955398758, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 128, "total_tokens": 152 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-string-number-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"007\" }", "pair": "grounded-string-number", "pairRole": "a", "resources": { "elapsed_ns": 511877777, "process_cpu_ns": 930503209, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 15, "prompt_tokens": 128, "total_tokens": 143 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-matching-record-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Nia\" }", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 541844727, "process_cpu_ns": 996603279, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-matching-record-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 1223838989, "process_cpu_ns": 2435888163, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-matching-record-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 1231938038, "process_cpu_ns": 2399452904, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 141, "total_tokens": 152 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-matching-record-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 1224150225, "process_cpu_ns": 2436330164, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-matching-record-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 1232349673, "process_cpu_ns": 2400227308, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 141, "total_tokens": 152 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-matching-record-a", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record,", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 528966010, "process_cpu_ns": 1041261652, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 141, "total_tokens": 165 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-matching-record-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"Nia\" }", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 553606324, "process_cpu_ns": 1020654819, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 141, "total_tokens": 155 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-matching-record-a", "model": "smol135-q4", "outcome": "scored", "output": "Here is a Python function that implements this approach:\n\n```python\ndef get_json_object(record, policy):\n \"\"\"\n Returns a JSON object with keys known and value.\n \"\"\"\n known = True\n for record in record:\n if known:\n return record\n else:\n known = False\n return None\n```\n\nThis function iterates through the records in the input list and checks if the field `known` is set to `True", "pair": "grounded-matching-record", "pairRole": "a", "resources": { "elapsed_ns": 1047867139, "process_cpu_ns": 2029384777, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 141, "total_tokens": 237 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-publish-consent-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": true,\n \"consent", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 861589075, "process_cpu_ns": 1705839947, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-publish-consent-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 688007252, "process_cpu_ns": 1317012070, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-publish-consent-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": true,\n \"consent\": true\n }\n}", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 977765490, "process_cpu_ns": 1933283195, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 30, "prompt_tokens": 70, "total_tokens": 100 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-publish-consent-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 689535645, "process_cpu_ns": 1320516390, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-publish-consent-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n \"\"\"\n Publish iff checks and consent are", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 371494503, "process_cpu_ns": 724801689, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-publish-consent-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 299339946, "process_cpu_ns": 539574162, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-publish-consent-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n \"\"\"\n Publish iff checks and consent are both true; otherwise hold. Return only a JSON object with string key decision.\n\n Args:\n record (dict): A dictionary of iff checks and consent.\n\n Returns:\n dict: A dictionary of iff checks and consent.\n \"\"\"\n if record[\"checks\"]:\n return {\n \"checks\": True,\n \"consent\": True\n ", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 849518535, "process_cpu_ns": 1634065934, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 70, "total_tokens": 166 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-publish-consent-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-consent", "pairRole": "a", "resources": { "elapsed_ns": 295594226, "process_cpu_ns": 533043389, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--typed-space-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \" \" }", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 606538815, "process_cpu_ns": 1169098182, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--typed-space-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \" \"\n}", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 600863623, "process_cpu_ns": 1194257889, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--typed-space-string", "model": "smol360-q4", "outcome": "scored", "output": "{ \"value\": \" \" }", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 615762559, "process_cpu_ns": 1187109916, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--typed-space-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"name\": \"my_payload\",\n \"type\": \"string", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 323335732, "process_cpu_ns": 631386254, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 65, "total_tokens": 89 } }, { "configuration": "short24", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--typed-space-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \" \" }", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 252587416, "process_cpu_ns": 461644185, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--typed-space-string", "model": "smol135-q4", "outcome": "scored", "output": "```json\n{\n \"payload\": {\n \"name\": \"my_payload\",\n \"type\": \"string\",\n \"value\": \"value\"\n }\n}\n```", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 431533754, "process_cpu_ns": 837529426, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 38, "prompt_tokens": 65, "total_tokens": 103 } }, { "configuration": "long96", "correct": true, "decoder": "schema", "family": "typed-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--typed-space-string", "model": "smol135-q4", "outcome": "scored", "output": "{ \"value\": \" \" }", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 257624140, "process_cpu_ns": 467701951, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 7, "prompt_tokens": 65, "total_tokens": 72 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--typed-space-string", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"payload\": \" \"\n}", "pair": "typed-pair-4", "pairRole": "b", "resources": { "elapsed_ns": 604820197, "process_cpu_ns": 1201639372, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 9, "prompt_tokens": 65, "total_tokens": 74 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--grounded-zero-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 1120590894, "process_cpu_ns": 2228864868, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 125, "total_tokens": 139 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--grounded-zero-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 1115351179, "process_cpu_ns": 2166183844, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--grounded-zero-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record):\n if", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 483647522, "process_cpu_ns": 950357592, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 125, "total_tokens": 149 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--grounded-zero-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"0\" }", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 480491172, "process_cpu_ns": 881319146, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 125, "total_tokens": 138 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--grounded-zero-a", "model": "smol135-q4", "outcome": "scored", "output": "Here's how you can implement this:\n\n```python\ndef get_json_object(record):\n if record.age == 0:\n return {\n \"value\": record.value,\n \"known\": True,\n \"known_fields\": [\n {\n \"name\": record.name,\n \"value\": record.value\n }\n ]\n }\n else:\n return {\n \"value\": record.value,\n \"known\":", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 1033664096, "process_cpu_ns": 1997376441, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 125, "total_tokens": 221 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--grounded-zero-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": \"0\" }", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 484457105, "process_cpu_ns": 889835603, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 13, "prompt_tokens": 125, "total_tokens": 138 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--unconstrained--grounded-zero-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"known\": true,\n \"value\": null\n}", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 1112027684, "process_cpu_ns": 2212219706, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 14, "prompt_tokens": 125, "total_tokens": 139 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "grounded-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--grounded-zero-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"known\": true, \"value\": null }", "pair": "grounded-zero", "pairRole": "a", "resources": { "elapsed_ns": 1126413967, "process_cpu_ns": 2188694167, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 11, "prompt_tokens": 125, "total_tokens": 136 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--schema--policy-publish-checks-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 686406407, "process_cpu_ns": 1313797086, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--short24--unconstrained--policy-publish-checks-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n \"\"\"\n Publish iff checks and consent are", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 334082009, "process_cpu_ns": 652986540, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--short24--schema--policy-publish-checks-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 297976962, "process_cpu_ns": 536187018, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol135-q4--long96--unconstrained--policy-publish-checks-a", "model": "smol135-q4", "outcome": "scored", "output": "```python\ndef publish_iff_check(record):\n \"\"\"\n Publish iff checks and consent are both true; otherwise hold. Return only a JSON object with string key decision.\n\n Args:\n record (dict): A dictionary of iff checks and consent.\n\n Returns:\n dict: A dictionary of iff checks and consent.\n \"\"\"\n if record[\"checks\"]:\n return {\n \"checks\": True,\n \"consent\": True\n ", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 878924909, "process_cpu_ns": 1688057018, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 96, "prompt_tokens": 70, "total_tokens": 166 } }, { "configuration": "long96", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol135-q4--long96--schema--policy-publish-checks-a", "model": "smol135-q4", "outcome": "scored", "output": "{ \"decision\": \"yes\" }", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 303130026, "process_cpu_ns": 545205849, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 10, "prompt_tokens": 70, "total_tokens": 80 } }, { "configuration": "short24", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": false, "id": "smol360-q4--short24--unconstrained--policy-publish-checks-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": true,\n \"consent", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 861657381, "process_cpu_ns": 1705307859, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 24, "prompt_tokens": 70, "total_tokens": 94 } }, { "configuration": "short24", "correct": false, "decoder": "schema", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--short24--schema--policy-publish-checks-a", "model": "smol360-q4", "outcome": "scored", "output": "{ \"decision\": \"Publish\" }", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 689552647, "process_cpu_ns": 1319665992, "process_maxrss_kib": 567192 }, "schemaValid": true, "tokens": { "completion_tokens": 9, "prompt_tokens": 70, "total_tokens": 79 } }, { "configuration": "long96", "correct": false, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": true, "id": "smol360-q4--long96--unconstrained--policy-publish-checks-a", "model": "smol360-q4", "outcome": "scored", "output": "{\n \"decision\": \"Publish\",\n \"record\": {\n \"checks\": true,\n \"consent\": true\n }\n}", "pair": "policy-publish-checks", "pairRole": "a", "resources": { "elapsed_ns": 982603513, "process_cpu_ns": 1941793033, "process_maxrss_kib": 567192 }, "schemaValid": false, "tokens": { "completion_tokens": 30, "prompt_tokens": 70, "total_tokens": 100 } } ], "custody": "Probity-operated; independent custody not established", "effects": "not executed", "elapsed_ns": 257401173568, "evidence": { "complete": true, "withinRunBudget": true }, "interpretation": "New finite authored boundary population; typed values, policy decisions and grounded known/unknown distinctions. Model and decoder selections were informed by prior public authored studies; no blinded or representative benchmark, real protected effect or independent custody claim. Format/schema correctness remain separate from strict semantic correctness.", "nativeTokens": { "completion": 7498, "prompt": 35280 }, "population": { "error": 0, "incomplete": 0, "planned": 384, "scored": 384, "started": 384, "unknown-start": 0, "unsupported": 0 }, "preparation": { "complete": true, "elapsedSeconds": 7.305208953000033, "scope": "retained transfer/source selection receipts reconstructed; weights/source wheels omitted from native packet and retained in full preparation archive; not an independent transfer or inference witness", "selectedPayloadBytes": 472221945, "selectedPayloadBytesByCategory": { "dependencies": 45457867, "metadata": 30738, "model": 376044704, "source": 50688636 } }, "process_cpu_ns": 497394631919, "profile": "probity-local-cpu-boundary-tasks-v1", "publicationDecision": "publish-scoped-report", "quality": [ { "callElapsedNs": 4723831541, "completionTokens": 310, "configuration": "short24", "correct": 0, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": 0, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 9240492026, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 5976613495, "completionTokens": 384, "configuration": "short24", "correct": 0, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": 0, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 11693640399, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 7814501471, "completionTokens": 384, "configuration": "short24", "correct": 0, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": 0, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 15363970770, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 4291221990, "completionTokens": 131, "configuration": "short24", "correct": 16, "decoder": "schema", "family": "typed-boundary", "formatValid": 16, "fullyCorrectPairs": 8, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 7754893355, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 5077534177, "completionTokens": 138, "configuration": "short24", "correct": 0, "decoder": "schema", "family": "policy-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 9286420480, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 7993651119, "completionTokens": 220, "configuration": "short24", "correct": 3, "decoder": "schema", "family": "grounded-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 14628609279, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 6497086579, "completionTokens": 553, "configuration": "long96", "correct": 0, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": 0, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 12617390214, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 10725354267, "completionTokens": 1041, "configuration": "long96", "correct": 0, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": 0, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 20732146066, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 14098939614, "completionTokens": 1231, "configuration": "long96", "correct": 0, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": 0, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 27341884632, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 4320565423, "completionTokens": 131, "configuration": "long96", "correct": 16, "decoder": "schema", "family": "typed-boundary", "formatValid": 16, "fullyCorrectPairs": 8, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 7797561488, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 5112517206, "completionTokens": 138, "configuration": "long96", "correct": 0, "decoder": "schema", "family": "policy-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 9348957832, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 7948825378, "completionTokens": 220, "configuration": "long96", "correct": 3, "decoder": "schema", "family": "grounded-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol135-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 14550379810, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 9954980498, "completionTokens": 159, "configuration": "short24", "correct": 0, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 19774843637, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 15309358990, "completionTokens": 384, "configuration": "short24", "correct": 0, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": 2, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 30337025043, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 17731268785, "completionTokens": 211, "configuration": "short24", "correct": 1, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 35279417494, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 10, "scored": 16 }, { "callElapsedNs": 10347515652, "completionTokens": 133, "configuration": "short24", "correct": 13, "decoder": "schema", "family": "typed-boundary", "formatValid": 16, "fullyCorrectPairs": 5, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 19844845665, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 12744687672, "completionTokens": 151, "configuration": "short24", "correct": 2, "decoder": "schema", "family": "policy-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 24487405820, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 18142441570, "completionTokens": 176, "configuration": "short24", "correct": 1, "decoder": "schema", "family": "grounded-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 35251934237, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 9897596024, "completionTokens": 159, "configuration": "long96", "correct": 0, "decoder": "unconstrained", "family": "typed-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 19661772937, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 18560132988, "completionTokens": 573, "configuration": "long96", "correct": 0, "decoder": "unconstrained", "family": "policy-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 36688759188, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 0, "scored": 16 }, { "callElapsedNs": 17766238167, "completionTokens": 211, "configuration": "long96", "correct": 1, "decoder": "unconstrained", "family": "grounded-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 35348355485, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 10, "scored": 16 }, { "callElapsedNs": 10294154970, "completionTokens": 133, "configuration": "long96", "correct": 13, "decoder": "schema", "family": "typed-boundary", "formatValid": 16, "fullyCorrectPairs": 5, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 19721071852, "processLifetimePeakRssKiB": 567192, "promptTokens": 1051, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 12591439115, "completionTokens": 151, "configuration": "long96", "correct": 2, "decoder": "schema", "family": "policy-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 24186369255, "processLifetimePeakRssKiB": 567192, "promptTokens": 1328, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 }, { "callElapsedNs": 18069864840, "completionTokens": 176, "configuration": "long96", "correct": 1, "decoder": "schema", "family": "grounded-boundary", "formatValid": 16, "fullyCorrectPairs": 0, "model": "smol360-q4", "planned": 16, "plannedPairs": 8, "processCpuNs": 35113092656, "processLifetimePeakRssKiB": 567192, "promptTokens": 2031, "resourceScope": "returned calls only; CPU is whole-process delta, RSS is lifetime peak, not task allocation", "schemaValid": 16, "scored": 16 } ], "tokenScope": "all returned native responses; errors have unknown token use" }, "schema": "probity-atlas-installed-boundary-report-v1", "upgrade": { "baseline-boundary": { "command": [ "/home/runner/work/_temp/format-consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/boundary-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/baseline-boundary", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-evidence-policy.json", "--policy-sha256", "493fe7fc7205681661fa89e3ab55689bc88723873854f2fd77fc46cbb599eef1" ], "exit": 1, "gate": { "evidenceDecision": "not-verified", "launched": false, "policySha256": "493fe7fc7205681661fa89e3ab55689bc88723873854f2fd77fc46cbb599eef1", "publicationDecision": "hold", "reason": "policy fields differ" } }, "baseline-comparison": { "command": [ "/home/runner/work/_temp/format-consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/comparison-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/baseline-comparison", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/comparison-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/comparison-evidence-policy.json", "--policy-sha256", "d22205e8f256a53d176dcdf6c7125855d6ff7218827dc728323bc8eb955dbf77" ], "exit": 0, "gate": { "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 67254, "sha256": "62fb95b5fff895ff76e4a42e17e391ec0525d3c5f9fade90efc64f5c0cad0892" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "d22205e8f256a53d176dcdf6c7125855d6ff7218827dc728323bc8eb955dbf77", "publicationDecision": "publish", "resources": { "completionTokens": 2539, "elapsedNs": 50885084703, "processLifetimePeakRssKiB": 535884, "promptTokens": 5656, "returnedCallProcessCpuNs": 98295793573 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "baseline-format": { "command": [ "/home/runner/work/_temp/format-consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/format-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/baseline-format", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/format-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/format-evidence-policy.json", "--policy-sha256", "b771ed50681ba68359b6d42cb52fd19dbf06d235799271e733681704e6fee73c" ], "exit": 0, "gate": { "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 142240, "sha256": "7cb54cc76384347736acdfa18c5e85427b60050f0da36b84a01d6ac9f110093f" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "b771ed50681ba68359b6d42cb52fd19dbf06d235799271e733681704e6fee73c", "publicationDecision": "publish", "resources": { "completionTokens": 3495, "elapsedNs": 93963267580, "processLifetimePeakRssKiB": 554152, "promptTokens": 11312, "returnedCallProcessCpuNs": 177947250784 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "baseline-installation": { "frameworkModules": { "agents": false, "google": false, "inspect_ai": false, "langgraph": false, "llama_cpp": false, "pydantic_ai": false }, "version": "0.0.3" }, "baseline-original": { "command": [ "/home/runner/work/_temp/format-consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/original-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/baseline-original", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/selected-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/evidence-policy.json", "--policy-sha256", "5c075f390e583d711f65908e11aeee4445c1012d4c55311e03e4faa88b8b39cc" ], "exit": 0, "gate": { "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 33853, "sha256": "acb8f021b5a9bda6b64e33ab38f9da592b8d2c2bba81de7bfdaef586256608f2" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "5c075f390e583d711f65908e11aeee4445c1012d4c55311e03e4faa88b8b39cc", "publicationDecision": "publish", "resources": { "completionTokens": 1840, "elapsedNs": 16723400890, "processLifetimePeakRssKiB": 200488, "promptTokens": 2828, "returnedCallProcessCpuNs": 31297972775 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "baseline-reader": { "command": [ "/home/runner/work/_temp/format-consumer/bin/probity-model-task-read", "/home/runner/work/_temp/model-boundary-upgrade-result/boundary-packet", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-native-pins.json", "--reader-sha256", "52a6000faae93178ce9e5acbf0e19c246fde69ee57055eb28cca8bddcc8f99b3" ], "exit": 1, "stdoutSha256": "c4e95bf792ecec38c196224f6d87d41fb73bad5088afa9fbf9c307bd3fb2f446" }, "boundaryOriginal": { "canonicalBytesEqualOriginal": true, "installedStdoutSha256": "9b9f2d5012a66ebd1ccb7a308ab859f0fed1da948fccc3915ae40ea79e9578c0", "originalMemberSha256": "fa7b0e39b69bcd42f82fc93096e071b534557c34cf0e03e73e1bac56dcdd72af" }, "candidate-boundary": { "command": [ "/home/runner/work/_temp/consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/boundary-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/candidate-boundary", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-evidence-policy.json", "--policy-sha256", "493fe7fc7205681661fa89e3ab55689bc88723873854f2fd77fc46cbb599eef1" ], "exit": 0, "gate": { "casePopulation": { "authoredIdentities": 48, "casesPerFamily": 16, "pairsPerFamily": 8, "uniqueLiteralInputs": 46 }, "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 291513, "sha256": "9b9f2d5012a66ebd1ccb7a308ab859f0fed1da948fccc3915ae40ea79e9578c0" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "493fe7fc7205681661fa89e3ab55689bc88723873854f2fd77fc46cbb599eef1", "publicationDecision": "publish", "resources": { "completionTokens": 7498, "elapsedNs": 257401173568, "preparationPayloadBytes": 472221945, "processLifetimePeakRssKiB": 567192, "promptTokens": 35280, "returnedCallProcessCpuNs": 496051239620, "totalProcessCpuNs": 497394631919 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "candidate-boundary-quality": { "command": [ "/home/runner/work/_temp/consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/boundary-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/candidate-boundary-quality", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-quality-policy.json", "--policy-sha256", "7e88df5bedbf75e633611e8eac4455e42e1cc6a86d1081deceb94cc85d50d50b" ], "exit": 1, "gate": { "casePopulation": { "authoredIdentities": 48, "casesPerFamily": 16, "pairsPerFamily": 8, "uniqueLiteralInputs": 46 }, "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 291513, "sha256": "9b9f2d5012a66ebd1ccb7a308ab859f0fed1da948fccc3915ae40ea79e9578c0" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [ { "field": "correct", "measured": 0, "minimum": 8, "row": { "configuration": "short24", "decoder": "schema", "family": "policy-boundary", "model": "smol135-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "short24", "decoder": "schema", "family": "policy-boundary", "model": "smol135-q4" } }, { "field": "correct", "measured": 3, "minimum": 8, "row": { "configuration": "short24", "decoder": "schema", "family": "grounded-boundary", "model": "smol135-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "short24", "decoder": "schema", "family": "grounded-boundary", "model": "smol135-q4" } }, { "field": "correct", "measured": 0, "minimum": 8, "row": { "configuration": "long96", "decoder": "schema", "family": "policy-boundary", "model": "smol135-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "long96", "decoder": "schema", "family": "policy-boundary", "model": "smol135-q4" } }, { "field": "correct", "measured": 3, "minimum": 8, "row": { "configuration": "long96", "decoder": "schema", "family": "grounded-boundary", "model": "smol135-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "long96", "decoder": "schema", "family": "grounded-boundary", "model": "smol135-q4" } }, { "field": "correct", "measured": 2, "minimum": 8, "row": { "configuration": "short24", "decoder": "schema", "family": "policy-boundary", "model": "smol360-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "short24", "decoder": "schema", "family": "policy-boundary", "model": "smol360-q4" } }, { "field": "correct", "measured": 1, "minimum": 8, "row": { "configuration": "short24", "decoder": "schema", "family": "grounded-boundary", "model": "smol360-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "short24", "decoder": "schema", "family": "grounded-boundary", "model": "smol360-q4" } }, { "field": "correct", "measured": 2, "minimum": 8, "row": { "configuration": "long96", "decoder": "schema", "family": "policy-boundary", "model": "smol360-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "long96", "decoder": "schema", "family": "policy-boundary", "model": "smol360-q4" } }, { "field": "correct", "measured": 1, "minimum": 8, "row": { "configuration": "long96", "decoder": "schema", "family": "grounded-boundary", "model": "smol360-q4" } }, { "field": "fullyCorrectPairs", "measured": 0, "minimum": 4, "row": { "configuration": "long96", "decoder": "schema", "family": "grounded-boundary", "model": "smol360-q4" } } ], "policySha256": "7e88df5bedbf75e633611e8eac4455e42e1cc6a86d1081deceb94cc85d50d50b", "publicationDecision": "hold-selected-score-or-resource", "resources": { "completionTokens": 7498, "elapsedNs": 257401173568, "preparationPayloadBytes": 472221945, "processLifetimePeakRssKiB": 567192, "promptTokens": 35280, "returnedCallProcessCpuNs": 496051239620, "totalProcessCpuNs": 497394631919 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "candidate-comparison": { "command": [ "/home/runner/work/_temp/consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/comparison-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/candidate-comparison", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/comparison-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/comparison-evidence-policy.json", "--policy-sha256", "d22205e8f256a53d176dcdf6c7125855d6ff7218827dc728323bc8eb955dbf77" ], "exit": 0, "gate": { "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 67254, "sha256": "62fb95b5fff895ff76e4a42e17e391ec0525d3c5f9fade90efc64f5c0cad0892" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "d22205e8f256a53d176dcdf6c7125855d6ff7218827dc728323bc8eb955dbf77", "publicationDecision": "publish", "resources": { "completionTokens": 2539, "elapsedNs": 50885084703, "processLifetimePeakRssKiB": 535884, "promptTokens": 5656, "returnedCallProcessCpuNs": 98295793573 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "candidate-format": { "command": [ "/home/runner/work/_temp/consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/format-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/candidate-format", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/format-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/format-evidence-policy.json", "--policy-sha256", "b771ed50681ba68359b6d42cb52fd19dbf06d235799271e733681704e6fee73c" ], "exit": 0, "gate": { "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 142240, "sha256": "7cb54cc76384347736acdfa18c5e85427b60050f0da36b84a01d6ac9f110093f" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "b771ed50681ba68359b6d42cb52fd19dbf06d235799271e733681704e6fee73c", "publicationDecision": "publish", "resources": { "completionTokens": 3495, "elapsedNs": 93963267580, "processLifetimePeakRssKiB": 554152, "promptTokens": 11312, "returnedCallProcessCpuNs": 177947250784 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "candidate-installation": { "frameworkModules": { "agents": false, "google": false, "inspect_ai": false, "langgraph": false, "llama_cpp": false, "pydantic_ai": false }, "version": "0.0.4" }, "candidate-mutant": { "command": [ "/home/runner/work/_temp/consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/mutant-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/candidate-mutant", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/boundary-evidence-policy.json", "--policy-sha256", "493fe7fc7205681661fa89e3ab55689bc88723873854f2fd77fc46cbb599eef1" ], "exit": 1, "gate": { "child": { "returncode": 1, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 69, "sha256": "11d7e04a543bc8ab088315d00daee1a2b505c0d2f01f7bcc8261c9bb229fe147" } }, "timedOut": false }, "evidenceDecision": "not-verified", "launched": true, "policySha256": "493fe7fc7205681661fa89e3ab55689bc88723873854f2fd77fc46cbb599eef1", "publicationDecision": "hold", "reason": "installed reader timed out or refused selected evidence" } }, "candidate-original": { "command": [ "/home/runner/work/_temp/consumer/bin/probity-model-publication-gate", "/home/runner/work/_temp/model-boundary-upgrade-result/original-packet", "/home/runner/work/_temp/model-boundary-upgrade-result/candidate-original", "--pins-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/selected-native-pins.json", "--policy-file", "/home/runner/work/agent-evidence-observer/agent-evidence-observer/interop/model-consumer-2026-10-02/evidence-policy.json", "--policy-sha256", "5c075f390e583d711f65908e11aeee4445c1012d4c55311e03e4faa88b8b39cc" ], "exit": 0, "gate": { "child": { "returncode": 0, "streams": { "reader.stderr": { "bytes": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "reader.stdout": { "bytes": 33853, "sha256": "acb8f021b5a9bda6b64e33ab38f9da592b8d2c2bba81de7bfdaef586256608f2" } }, "timedOut": false }, "evidenceDecision": "verified", "launched": true, "policyFailures": [], "policySha256": "5c075f390e583d711f65908e11aeee4445c1012d4c55311e03e4faa88b8b39cc", "publicationDecision": "publish", "resources": { "completionTokens": 1840, "elapsedNs": 16723400890, "processLifetimePeakRssKiB": 200488, "promptTokens": 2828, "returnedCallProcessCpuNs": 31297972775 }, "scope": "host-selected report publication; no dispatch or recovery authority" } }, "preservedReports": { "comparison": "62fb95b5fff895ff76e4a42e17e391ec0525d3c5f9fade90efc64f5c0cad0892", "format": "7cb54cc76384347736acdfa18c5e85427b60050f0da36b84a01d6ac9f110093f", "original": "acb8f021b5a9bda6b64e33ab38f9da592b8d2c2bba81de7bfdaef586256608f2" } } }