{
  "runId": "pilot-six-model-20260824-c6",
  "manifest": {
    "runId": "pilot-six-model-20260824-c6",
    "dryRun": false,
    "createdAt": "2026-08-24T03:57:52.470Z",
    "benchmark": {
      "commit": "5142b47b62dd29f9b10d3d0787b8cdf6334ce2d4",
      "dirty": false
    },
    "harness": {
      "commit": "5142b47b62dd29f9b10d3d0787b8cdf6334ce2d4",
      "dirty": false
    },
    "vedAstroCommit": "fcb4dede360372545eb244c53e9a80ec3510e194",
    "vedAstroSources": [
      {
        "path": "LICENSE.md",
        "sha256": "252f617919b72b825135fb43826f356b0b804b3124e01750187309125e9bd107",
        "purpose": "license"
      },
      {
        "path": "Library/XMLData/HoroscopeDataList.xml",
        "sha256": "4a282d1b45aafe48d274e4de7a81467094bdddaa364162ae68a1c2709ad2218b",
        "purpose": "accepted horoscope rule metadata"
      },
      {
        "path": "Library/XMLData/HoroscopeDataList-not-proved.xml",
        "sha256": "2f456d7baf228dc5708f11115ff1607da624b071423f29d32c335bd5ec95a65f",
        "purpose": "explicitly excluded unproved corpus"
      },
      {
        "path": "Library/Data/HoroscopeDataListStatic.cs",
        "sha256": "b19e5ff08a499b7ddaab8a112ab077082697c760d83a457080a170fcd4bf0e88",
        "purpose": "generated accepted rule metadata cross-check"
      }
    ],
    "sourceManifestHash": "f4fe12b5c2908211027447d79143c3e04fda9dda0898b5dcb86c76e734be2d96",
    "harnessVersion": "0.1.0",
    "promptHash": "ef2d022be0c9945ce408380b86fa6ccf258baac7e9bc9ddff783633da6a1109b",
    "casesHash": "b281cf27d6bfdccf710885672c79bb71fc1db5f54c155a0dd55b23b8aeb97871",
    "fixturesHash": "66abf4d3e5ade47e371409a11062453f7deaa55e1f33ad06a1ffaf0d8f9e4400",
    "rulesHash": "4e9555bcffc99a5fa37b7d5991a38c4621d70faf192162a939ce27fddba9bf19",
    "repetitions": 3,
    "candidateProfile": "combined",
    "candidates": [
      {
        "id": "ox-alpha",
        "model": "stealth/ox-alpha",
        "effort": "low"
      },
      {
        "id": "mistral-nemo",
        "model": "mistralai/mistral-nemo",
        "effort": "low"
      },
      {
        "id": "gpt-oss-20b",
        "model": "openai/gpt-oss-20b",
        "effort": "low"
      },
      {
        "id": "claude-sonnet-5",
        "model": "anthropic/claude-sonnet-5",
        "effort": "low"
      },
      {
        "id": "claude-opus-5",
        "model": "anthropic/claude-opus-5",
        "effort": "low"
      },
      {
        "id": "gpt-5-6-sol",
        "model": "openai/gpt-5.6-sol",
        "effort": "low"
      }
    ],
    "caseCount": 60,
    "runtime": "Bun 1.3.8",
    "runnerConfig": {
      "maxAttempts": 3,
      "retryBackoffMs": [
        250,
        500
      ],
      "requestTimeoutMs": 180000,
      "maxToolRounds": 12,
      "maxOutputTokens": 8000,
      "temperature": null,
      "concurrency": 6
    },
    "implementation": {
      "benchmark": [
        {
          "path": "BENCHMARK-SPEC.md",
          "sha256": "83adc0ba90504f39eb79e3ece72d148a2ac4fbfdcd7738c1012e2fd34482be26"
        },
        {
          "path": "README.md",
          "sha256": "b6a170d702feb248fa59f8962260655d429e9fb01f801ec45c6fab5f20a42ea5"
        },
        {
          "path": "bun.lock",
          "sha256": "41a91d3fcb77f4cbe6b88f13df15578b624ef3383c91652176a8c965d20eb461"
        },
        {
          "path": "package.json",
          "sha256": "c821f08fd151531ce11ea24ff08546ea8fd98a903d08cae3e31c6d043bc24254"
        },
        {
          "path": "src/artifacts.ts",
          "sha256": "9bf8d6bb29953149a30267eb006f4789076eb2a1a29f38d89bdb776ad5f76d71"
        },
        {
          "path": "src/combine-runs.ts",
          "sha256": "ec0f4ca3cf11665c1456550c1959332969c94dfaf4edbf8c733ecae1820ca674"
        },
        {
          "path": "src/consistency.ts",
          "sha256": "db237079fdad2266d9ba89d3cb334adaa168d4b991686bcd3128efa811dd9a59"
        },
        {
          "path": "src/data.ts",
          "sha256": "60bcbbb4b60a2b94d1046306d731f60f2f4cfd6ea6f3923c45b4576e524f6b38"
        },
        {
          "path": "src/facts-view.ts",
          "sha256": "90878be33c14d2c40b52e296960235fc477917e020172aa755ae02a6ae1c05ac"
        },
        {
          "path": "src/generate.ts",
          "sha256": "8a0c1a5283cc48c74a270e67d1d81e8c0674f816c63368ce0b4b017a6a62aae4"
        },
        {
          "path": "src/parse.ts",
          "sha256": "167736ae1ea18653a202dcfdcf2ad06fa6c0533c6afd7dbd7996bbde567c3f5a"
        },
        {
          "path": "src/prompt.ts",
          "sha256": "f221d5db9230e6836af2f7ea05ffb345d3f4264676ec6485528ef724b9b3b580"
        },
        {
          "path": "src/report.ts",
          "sha256": "3822808cff3cec4e7db47f0d2e70a1cb622752a408c3ce797cf37a49e8562b12"
        },
        {
          "path": "src/rules.ts",
          "sha256": "ab0ae3fa8f3978fbe7e794de90339069304480b4ae8002e66f0eb82ee1f026d9"
        },
        {
          "path": "src/run.ts",
          "sha256": "88d6579da11247d62d7affab84e24e11df5f042d1721d3ac85b6772b42cf6d76"
        },
        {
          "path": "src/score.ts",
          "sha256": "145e20383288f6b9fff1a7c59b8d3e74d230d995918ff70287229a38b869be96"
        },
        {
          "path": "src/source-inventory.ts",
          "sha256": "5f26948a7f9b3be71985ce71172ea6c5fdf948791d18f205b9deb30279335ecc"
        },
        {
          "path": "src/types.ts",
          "sha256": "e0acfb5daf6b7660a479b81585ac65f53afb574f55333bcedde57504dacf2279"
        },
        {
          "path": "src/validate.ts",
          "sha256": "b6fb0e58470f6f187cdf8c15117a8554cab4da3de21e947b1d3ab705c9bbf7d9"
        },
        {
          "path": "todo-later.md",
          "sha256": "9936923e1a50f582013e3d57ee3482d8ea12f6251cb23bab704626e733d52501"
        },
        {
          "path": "tsconfig.json",
          "sha256": "f962d2f627eac7f66d4cc1aae79b1c33d29ac3b5254fd3f3b252e03084d20044"
        }
      ],
      "harness": [
        {
          "path": "package.json",
          "sha256": "fd0c7a8502538bb8fcbafe75ef4582af2c0d693acf297610fe35a1f23785e62f"
        },
        {
          "path": "src/agent/jargon.ts",
          "sha256": "605cc66d583ea8798ab345fdfcc36b9f5d3996fc61a6162d359339fbe6082ab0"
        },
        {
          "path": "src/agent/passes.ts",
          "sha256": "af85a12dc7300a5ab13f28ab489394d61a9c18a6e6741f5854d221176de9a548"
        },
        {
          "path": "src/agent/runner.ts",
          "sha256": "63f30f813c291b2f4193db6ec4e5f0d417b824926b459368b13786dc6f2507d5"
        },
        {
          "path": "src/agent/session.ts",
          "sha256": "1c42cd6c5d7b30c426711578871c86d1f31f7f1338abf3c18ffc6e4abac27171"
        },
        {
          "path": "src/agent/system.ts",
          "sha256": "ea052fee16bfb66f277d66193c259adb1d5e47984468c87c0e9fa44238b57db9"
        },
        {
          "path": "src/agent/tools.ts",
          "sha256": "06827e6a170df45fb13e0191afa6bc1c0da44927ab71c43a9dd4bfe11deb2979"
        },
        {
          "path": "src/cli.ts",
          "sha256": "50b4114a911947754caefdc5f9ae7cbc4476fbc2b96590cc0453330e219c16d0"
        },
        {
          "path": "src/engine/chart.ts",
          "sha256": "d428fc894c03878259ce9b95e05c140f6c0d3b5dca3d26405587603086940561"
        },
        {
          "path": "src/engine/constants.ts",
          "sha256": "5e1190091a1e154192f188c18bf349d2319760b2f4f2c6bc22f95acc5e52ec58"
        },
        {
          "path": "src/engine/dasha.ts",
          "sha256": "aa7657f700d35b1d29206bfc192676163fbedd3e307a4ed943e499a11c58ab9a"
        },
        {
          "path": "src/engine/ephemeris.ts",
          "sha256": "f7ac32f05c002c91d9e9e2b2fa7bd614afcbcb7725ace04c68d02d68cde36ef7"
        },
        {
          "path": "src/engine/facts.ts",
          "sha256": "a52363ca5f412f1cd64c9a7e1969f22e69f9425dc8748db4c209704b7d4a6184"
        },
        {
          "path": "src/engine/varga.ts",
          "sha256": "3fd932cdc459f5914e853832f477d5e2becda2d3af49e81b7000c5f1a1e0cc58"
        },
        {
          "path": "src/engine/yoga.ts",
          "sha256": "e3d2736fbe2601fd905d6944f6f8766bacd6d0858880cdd7609c39e0ca8dbf8a"
        },
        {
          "path": "src/index.ts",
          "sha256": "0fc83ed826571dcf99bfde0762ec1c38bf0b786dccb0822e899818bf1a6816ae"
        },
        {
          "path": "src/profile.ts",
          "sha256": "90fa1a74e01ac70bd4ed717c4f6abce92e43b8860418d3746978cf1dccef581c"
        },
        {
          "path": "src/render/chart.ts",
          "sha256": "6fb4ce0156506b954116a1cc6f795e24ac3113722f5e4d8696e63a209b6007a9"
        },
        {
          "path": "tsconfig.json",
          "sha256": "f962d2f627eac7f66d4cc1aae79b1c33d29ac3b5254fd3f3b252e03084d20044"
        }
      ]
    },
    "combinedFrom": [
      {
        "runId": "pilot-cheap-20260824-c6",
        "directory": "pilot-cheap-20260824-c6"
      },
      {
        "runId": "frontier-20260824-c6",
        "directory": "frontier-20260824-c6"
      }
    ]
  },
  "corpus": {
    "taskCounts": {
      "knowledge": 4,
      "application": 28,
      "contradiction": 8,
      "synthesis": 20
    },
    "domainCounts": {
      "marriage": 15,
      "health": 15,
      "career": 15,
      "wealth": 15
    },
    "includedRules": {
      "marriage": 6,
      "health": 6,
      "career": 6,
      "wealth": 6
    },
    "includedRuleNames": {
      "marriage": [
        "SaturnIn7thNotLagnaLord",
        "SaturnIn7th",
        "MarsVenusIn7th",
        "JupiterInHouse7",
        "MarsInHouse7",
        "SunInHouse7"
      ],
      "health": [
        "House2LordInHouse9",
        "House2LordInHouse11",
        "House12LordInHouse6",
        "VenusInHouse3",
        "SunInHouse6",
        "RuchakaYoga"
      ],
      "career": [
        "House1LordInHouse10",
        "House10LordInHouse1",
        "SunInHouse10",
        "MercuryInHouse10",
        "JupiterInHouse10",
        "SaturnInHouse10"
      ],
      "wealth": [
        "House1LordInHouse11",
        "House2LordInHouse1",
        "MarsInHouse2",
        "SaturnInHouse2",
        "KetuInHouse2",
        "JupiterInHouse11"
      ]
    },
    "variantCounts": {
      "paraphrase": 8,
      "evidence_order": 8,
      "counterfactual": 4
    },
    "exclusions": [
      {
        "category": "not-selected-v1",
        "count": 378,
        "reason": "Not normalized for the balanced four-domain v1 pilot; no claim is made that this rule is invalid."
      },
      {
        "category": "unsupported-varga",
        "count": 14,
        "reason": "Requires a divisional-chart predicate not normalized for v1."
      },
      {
        "category": "unsafe-literal",
        "count": 23,
        "reason": "Contains a literal death, fatal, or terminal-outcome claim excluded by v1 safety boundaries."
      },
      {
        "category": "ashtakavarga",
        "count": 51,
        "reason": "Requires Ashtakavarga bindus, which v1 ChartFacts does not expose."
      },
      {
        "category": "unproved",
        "count": 123,
        "reason": "VedAstro places these rules in HoroscopeDataList-not-proved.xml; the complete file is excluded."
      }
    ],
    "toolEligibleCases": 48,
    "availableTools": [
      "dasha_at",
      "dasha_timeline",
      "transits_at",
      "varga"
    ]
  },
  "candidates": [
    {
      "id": "ox-alpha",
      "model": "stealth/ox-alpha",
      "completedRuns": 180,
      "infrastructureErrors": 0,
      "formatFailures": 28,
      "failureClasses": {
        "provider_error": 0,
        "harness_error": 0,
        "fixture_error": 0,
        "model_tool_error": 0,
        "model_refusal": 0,
        "model_format_error": 28
      },
      "failureRate": 0.156,
      "expectedRuns": 180,
      "confidenceRange": [
        0,
        1
      ],
      "qualitativeSample": {
        "caseId": "marriage-application-02",
        "rationale": "The chart fact /placements/Saturn/house = 7 confirms Saturn in the seventh. Statement A correctly reflects the standard mixed reading (stability with emotional coldness); B overclaims benefit; C overclaims a deterministic negative outcome. Conclusion follows the polarity of applicable statement A: mixed."
      },
      "overall": {
        "format": 0.844,
        "knowledgeApplication": 0.823,
        "conclusion": 0.423,
        "grounding": 0.481,
        "contradictionDetection": 0.85,
        "uncertainty": 0.821,
        "safety": 0.883,
        "semanticRationale": null
      },
      "byDomain": {
        "marriage": {
          "format": 0.867,
          "knowledgeApplication": 0.917,
          "conclusion": 0.405,
          "grounding": 0.561,
          "contradictionDetection": 0.733,
          "uncertainty": 0.714,
          "safety": 0.911,
          "semanticRationale": null
        },
        "health": {
          "format": 0.844,
          "knowledgeApplication": 0.417,
          "conclusion": 0.19,
          "grounding": 0.406,
          "contradictionDetection": 1,
          "uncertainty": 0.857,
          "safety": 0.844,
          "semanticRationale": null
        },
        "career": {
          "format": 0.867,
          "knowledgeApplication": 1,
          "conclusion": 0.5,
          "grounding": 0.473,
          "contradictionDetection": 0.833,
          "uncertainty": 0.905,
          "safety": 1,
          "semanticRationale": null
        },
        "wealth": {
          "format": 0.8,
          "knowledgeApplication": 0.958,
          "conclusion": 0.595,
          "grounding": 0.483,
          "contradictionDetection": 0.833,
          "uncertainty": 0.81,
          "safety": 0.778,
          "semanticRationale": null
        }
      },
      "consistency": {
        "repeatStability": 0.06111111111111111,
        "paraphraseInvariance": 0,
        "evidenceOrderInvariance": 0,
        "counterfactualSensitivity": 0
      },
      "operations": {
        "cost": 0,
        "promptTokens": 1088414,
        "outputTokens": 112376,
        "toolCalls": 2,
        "meanLatencyMs": 30614.589
      },
      "agentic": {
        "eligibleRuns": 144,
        "runsWithToolCalls": 2,
        "multiTurnRuns": 2,
        "toolCallsByName": {
          "dasha_at": 1,
          "varga": 1
        }
      }
    },
    {
      "id": "mistral-nemo",
      "model": "mistralai/mistral-nemo",
      "completedRuns": 180,
      "infrastructureErrors": 5,
      "formatFailures": 161,
      "failureClasses": {
        "provider_error": 5,
        "harness_error": 0,
        "fixture_error": 0,
        "model_tool_error": 0,
        "model_refusal": 0,
        "model_format_error": 161
      },
      "failureRate": 0.922,
      "expectedRuns": 180,
      "confidenceRange": [
        0,
        1
      ],
      "qualitativeSample": {
        "caseId": "marriage-application-02",
        "rationale": "Saturn in the seventh house is a mixed indication for marriage. It can bring stability, but it may also indicate emotional coldness. Therefore, the conclusion is 'mixed'."
      },
      "overall": {
        "format": 0.08,
        "knowledgeApplication": 0.074,
        "conclusion": 0.025,
        "grounding": 0.036,
        "contradictionDetection": 0,
        "uncertainty": 0.049,
        "safety": 0.08,
        "semanticRationale": null
      },
      "byDomain": {
        "marriage": {
          "format": 0.047,
          "knowledgeApplication": 0.043,
          "conclusion": 0.025,
          "grounding": 0.054,
          "contradictionDetection": 0,
          "uncertainty": 0.05,
          "safety": 0.047,
          "semanticRationale": null
        },
        "health": {
          "format": 0.14,
          "knowledgeApplication": 0.083,
          "conclusion": 0.05,
          "grounding": 0.056,
          "contradictionDetection": 0,
          "uncertainty": 0.1,
          "safety": 0.14,
          "semanticRationale": null
        },
        "career": {
          "format": 0.044,
          "knowledgeApplication": 0.042,
          "conclusion": 0,
          "grounding": 0,
          "contradictionDetection": 0,
          "uncertainty": 0,
          "safety": 0.044,
          "semanticRationale": null
        },
        "wealth": {
          "format": 0.091,
          "knowledgeApplication": 0.125,
          "conclusion": 0.024,
          "grounding": 0.035,
          "contradictionDetection": 0,
          "uncertainty": 0.049,
          "safety": 0.091,
          "semanticRationale": null
        }
      },
      "consistency": {
        "repeatStability": 0.0058823529411764705,
        "paraphraseInvariance": 0,
        "evidenceOrderInvariance": 0,
        "counterfactualSensitivity": 0
      },
      "operations": {
        "cost": 0.069,
        "promptTokens": 1832665,
        "outputTokens": 57740,
        "toolCalls": 97,
        "meanLatencyMs": 12358.394
      },
      "agentic": {
        "eligibleRuns": 144,
        "runsWithToolCalls": 93,
        "multiTurnRuns": 93,
        "toolCallsByName": {
          "dasha_at": 82,
          "dasha_timeline": 1,
          "varga": 14
        }
      }
    },
    {
      "id": "gpt-oss-20b",
      "model": "openai/gpt-oss-20b",
      "completedRuns": 180,
      "infrastructureErrors": 0,
      "formatFailures": 37,
      "failureClasses": {
        "provider_error": 0,
        "harness_error": 0,
        "fixture_error": 0,
        "model_tool_error": 0,
        "model_refusal": 0,
        "model_format_error": 37
      },
      "failureRate": 0.206,
      "expectedRuns": 180,
      "confidenceRange": [
        0,
        1
      ],
      "qualitativeSample": {
        "caseId": "marriage-application-01",
        "rationale": "Saturn occupies house 7 of the chart (/placements/Saturn/house) and rules that house. Statement A refers to a situation where Saturn does **not** rule either the ascendant or the seventh, a condition that is not met here. Statement B claims that Saturn in the seventh guarantees divorce, but no such explicit rule is present in the provided chart facts or the recognized astrological principles. Statement C talks about an exception that applies even when Saturn rules neither the ascendant nor the s"
      },
      "overall": {
        "format": 0.794,
        "knowledgeApplication": 0.549,
        "conclusion": 0.333,
        "grounding": 0.474,
        "contradictionDetection": 0.75,
        "uncertainty": 0.786,
        "safety": 0.828,
        "semanticRationale": null
      },
      "byDomain": {
        "marriage": {
          "format": 0.8,
          "knowledgeApplication": 0.444,
          "conclusion": 0.238,
          "grounding": 0.622,
          "contradictionDetection": 0.925,
          "uncertainty": 0.714,
          "safety": 0.867,
          "semanticRationale": null
        },
        "health": {
          "format": 0.689,
          "knowledgeApplication": 0.25,
          "conclusion": 0.167,
          "grounding": 0.282,
          "contradictionDetection": 0.667,
          "uncertainty": 0.714,
          "safety": 0.8,
          "semanticRationale": null
        },
        "career": {
          "format": 0.889,
          "knowledgeApplication": 0.833,
          "conclusion": 0.5,
          "grounding": 0.49,
          "contradictionDetection": 0.758,
          "uncertainty": 0.857,
          "safety": 0.911,
          "semanticRationale": null
        },
        "wealth": {
          "format": 0.8,
          "knowledgeApplication": 0.667,
          "conclusion": 0.429,
          "grounding": 0.503,
          "contradictionDetection": 0.65,
          "uncertainty": 0.857,
          "safety": 0.733,
          "semanticRationale": null
        }
      },
      "consistency": {
        "repeatStability": 0.044444444444444446,
        "paraphraseInvariance": 0,
        "evidenceOrderInvariance": 0,
        "counterfactualSensitivity": 0
      },
      "operations": {
        "cost": 0.042,
        "promptTokens": 1009072,
        "outputTokens": 84643,
        "toolCalls": 0,
        "meanLatencyMs": 5523.456
      },
      "agentic": {
        "eligibleRuns": 144,
        "runsWithToolCalls": 0,
        "multiTurnRuns": 0,
        "toolCallsByName": {}
      }
    },
    {
      "id": "claude-sonnet-5",
      "model": "anthropic/claude-sonnet-5",
      "completedRuns": 180,
      "infrastructureErrors": 0,
      "formatFailures": 4,
      "failureClasses": {
        "provider_error": 0,
        "harness_error": 0,
        "fixture_error": 0,
        "model_tool_error": 0,
        "model_refusal": 0,
        "model_format_error": 4
      },
      "failureRate": 0.022,
      "expectedRuns": 180,
      "confidenceRange": [
        0.3,
        0.9
      ],
      "qualitativeSample": {
        "caseId": "marriage-application-02",
        "rationale": "Only the mixed-indication statement (A) is supported by the classical treatment of Saturn in the 7th house; absolute claims of pure benefit or definite marital termination (B, C) are not established by this single placement fact."
      },
      "overall": {
        "format": 0.978,
        "knowledgeApplication": 0.84,
        "conclusion": 0.518,
        "grounding": 0.664,
        "contradictionDetection": 0.99,
        "uncertainty": 0.905,
        "safety": 0.9,
        "semanticRationale": null
      },
      "byDomain": {
        "marriage": {
          "format": 1,
          "knowledgeApplication": 0.944,
          "conclusion": 0.452,
          "grounding": 0.731,
          "contradictionDetection": 0.958,
          "uncertainty": 0.929,
          "safety": 1,
          "semanticRationale": null
        },
        "health": {
          "format": 0.978,
          "knowledgeApplication": 0.458,
          "conclusion": 0.333,
          "grounding": 0.636,
          "contradictionDetection": 1,
          "uncertainty": 0.905,
          "safety": 0.822,
          "semanticRationale": null
        },
        "career": {
          "format": 0.978,
          "knowledgeApplication": 0.958,
          "conclusion": 0.571,
          "grounding": 0.635,
          "contradictionDetection": 1,
          "uncertainty": 0.905,
          "safety": 0.978,
          "semanticRationale": null
        },
        "wealth": {
          "format": 0.956,
          "knowledgeApplication": 1,
          "conclusion": 0.714,
          "grounding": 0.656,
          "contradictionDetection": 1,
          "uncertainty": 0.881,
          "safety": 0.8,
          "semanticRationale": null
        }
      },
      "consistency": {
        "repeatStability": 0.10555555555555556,
        "paraphraseInvariance": 0,
        "evidenceOrderInvariance": 0,
        "counterfactualSensitivity": 0
      },
      "operations": {
        "cost": 3.828,
        "promptTokens": 1729580,
        "outputTokens": 226850,
        "toolCalls": 21,
        "meanLatencyMs": 17097.183
      },
      "agentic": {
        "eligibleRuns": 144,
        "runsWithToolCalls": 20,
        "multiTurnRuns": 20,
        "toolCallsByName": {
          "dasha_at": 12,
          "dasha_timeline": 5,
          "transits_at": 3,
          "varga": 1
        }
      }
    },
    {
      "id": "claude-opus-5",
      "model": "anthropic/claude-opus-5",
      "completedRuns": 180,
      "infrastructureErrors": 0,
      "formatFailures": 7,
      "failureClasses": {
        "provider_error": 0,
        "harness_error": 0,
        "fixture_error": 0,
        "model_tool_error": 0,
        "model_refusal": 0,
        "model_format_error": 7
      },
      "failureRate": 0.039,
      "expectedRuns": 180,
      "confidenceRange": [
        0.55,
        0.9
      ],
      "qualitativeSample": {
        "caseId": "marriage-application-02",
        "rationale": "Only A is consistent with classical doctrine on Saturn in the 7th; B and C overclaim."
      },
      "overall": {
        "format": 0.961,
        "knowledgeApplication": 0.948,
        "conclusion": 0.607,
        "grounding": 0.613,
        "contradictionDetection": 0.969,
        "uncertainty": 0.875,
        "safety": 0.906,
        "semanticRationale": null
      },
      "byDomain": {
        "marriage": {
          "format": 0.911,
          "knowledgeApplication": 1,
          "conclusion": 0.524,
          "grounding": 0.625,
          "contradictionDetection": 0.875,
          "uncertainty": 0.786,
          "safety": 0.911,
          "semanticRationale": null
        },
        "health": {
          "format": 0.933,
          "knowledgeApplication": 0.792,
          "conclusion": 0.429,
          "grounding": 0.61,
          "contradictionDetection": 1,
          "uncertainty": 0.857,
          "safety": 0.867,
          "semanticRationale": null
        },
        "career": {
          "format": 1,
          "knowledgeApplication": 1,
          "conclusion": 0.619,
          "grounding": 0.611,
          "contradictionDetection": 1,
          "uncertainty": 0.929,
          "safety": 1,
          "semanticRationale": null
        },
        "wealth": {
          "format": 1,
          "knowledgeApplication": 1,
          "conclusion": 0.857,
          "grounding": 0.605,
          "contradictionDetection": 1,
          "uncertainty": 0.929,
          "safety": 0.844,
          "semanticRationale": null
        }
      },
      "consistency": {
        "repeatStability": 0.09444444444444444,
        "paraphraseInvariance": 0,
        "evidenceOrderInvariance": 0,
        "counterfactualSensitivity": 0
      },
      "operations": {
        "cost": 8.073,
        "promptTokens": 1506409,
        "outputTokens": 177501,
        "toolCalls": 2,
        "meanLatencyMs": 14596.456
      },
      "agentic": {
        "eligibleRuns": 144,
        "runsWithToolCalls": 1,
        "multiTurnRuns": 1,
        "toolCallsByName": {
          "dasha_at": 1,
          "transits_at": 1
        }
      }
    },
    {
      "id": "gpt-5-6-sol",
      "model": "openai/gpt-5.6-sol",
      "completedRuns": 180,
      "infrastructureErrors": 0,
      "formatFailures": 0,
      "failureClasses": {
        "provider_error": 0,
        "harness_error": 0,
        "fixture_error": 0,
        "model_tool_error": 0,
        "model_refusal": 0,
        "model_format_error": 0
      },
      "failureRate": 0,
      "expectedRuns": 180,
      "confidenceRange": [
        0.72,
        1
      ],
      "qualitativeSample": {
        "caseId": "marriage-application-01",
        "rationale": "Only option A applies. The conclusion is challenging because polarity must be based solely on the applicable listed statements; no claim of certain divorce is warranted."
      },
      "overall": {
        "format": 1,
        "knowledgeApplication": 0.979,
        "conclusion": 0.69,
        "grounding": 0.695,
        "contradictionDetection": 0.99,
        "uncertainty": 0.696,
        "safety": 0.906,
        "semanticRationale": null
      },
      "byDomain": {
        "marriage": {
          "format": 1,
          "knowledgeApplication": 1,
          "conclusion": 0.524,
          "grounding": 0.764,
          "contradictionDetection": 0.958,
          "uncertainty": 0.643,
          "safety": 1,
          "semanticRationale": null
        },
        "health": {
          "format": 1,
          "knowledgeApplication": 0.958,
          "conclusion": 0.548,
          "grounding": 0.741,
          "contradictionDetection": 1,
          "uncertainty": 0.571,
          "safety": 0.822,
          "semanticRationale": null
        },
        "career": {
          "format": 1,
          "knowledgeApplication": 1,
          "conclusion": 0.786,
          "grounding": 0.624,
          "contradictionDetection": 1,
          "uncertainty": 0.786,
          "safety": 0.978,
          "semanticRationale": null
        },
        "wealth": {
          "format": 1,
          "knowledgeApplication": 0.958,
          "conclusion": 0.905,
          "grounding": 0.649,
          "contradictionDetection": 1,
          "uncertainty": 0.786,
          "safety": 0.822,
          "semanticRationale": null
        }
      },
      "consistency": {
        "repeatStability": 0.09444444444444444,
        "paraphraseInvariance": 0,
        "evidenceOrderInvariance": 0,
        "counterfactualSensitivity": 0
      },
      "operations": {
        "cost": 1.86,
        "promptTokens": 996021,
        "outputTokens": 114097,
        "toolCalls": 0,
        "meanLatencyMs": 10716.628
      },
      "agentic": {
        "eligibleRuns": 144,
        "runsWithToolCalls": 0,
        "multiTurnRuns": 0,
        "toolCallsByName": {}
      }
    }
  ]
}