{
  "mode": "live",
  "phase": "pilot",
  "bundleHash": "1a7bcb1a54e0578b18117d22d0b21745a4b85008c605343967463e6bf4e5d15a",
  "responses": 720,
  "brandLabels": 2160,
  "primary": [
    {
      "contrast": "sol_minus_luna",
      "difference": -0.012152777777777781,
      "ci95": [
        -0.04513888888888889,
        0.02256944444444443
      ],
      "p": 0.5057557609320958,
      "holmP": 0.5057557609320958,
      "categories": 12
    },
    {
      "contrast": "astra_minus_sol",
      "difference": -0.07638888888888891,
      "ci95": [
        -0.12673611111111116,
        -0.026041666666666657
      ],
      "p": 0.01695226550716669,
      "holmP": 0.06780906202866677,
      "categories": 12
    },
    {
      "contrast": "search_luna",
      "difference": -0.06944444444444443,
      "ci95": [
        -0.13194444444444442,
        -9.25185853854297e-18
      ],
      "p": 0.07743486013452355,
      "holmP": 0.23230458040357066,
      "categories": 12
    },
    {
      "contrast": "search_sol",
      "difference": -0.07986111111111109,
      "ci95": [
        -0.15277777777777776,
        0.0034722222222222654
      ],
      "p": 0.09056749934449494,
      "holmP": 0.23230458040357066,
      "categories": 12
    },
    {
      "contrast": "search_astra",
      "difference": -0.07291666666666664,
      "ci95": [
        -0.11805555555555554,
        -0.027777777777777766
      ],
      "p": 0.011603466668357092,
      "holmP": 0.05801733334178546,
      "categories": 12
    }
  ],
  "gatewayExploratory": {
    "label": "EXPLORATORY matched gateway subset; no confirmatory p value",
    "categories": 12,
    "excludedIncompleteCategories": [],
    "openrouterMinusDirect": -0.009259259259259259,
    "categoryDifferences": [
      0.027777777777777776,
      -0.05555555555555555,
      -0.05555555555555555,
      0.0,
      -0.027777777777777776,
      0.05555555555555555,
      -0.1111111111111111,
      0.05555555555555555,
      0.027777777777777776,
      0.0,
      0.0,
      -0.027777777777777776
    ]
  },
  "conditions": [
    {
      "condition": "direct-astra-none",
      "responses": 96,
      "brandLabels": 288,
      "mentionRate": 0.6805555555555556,
      "accountedUsd": 3.56635,
      "costBasis": [
        "conservative_usage_estimate"
      ],
      "medianLatencySeconds": 21.0,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 0,
      "parsedSearchCountRange": [
        0,
        0
      ],
      "repeatPairs": 48,
      "differentPanelBrandSetPairs": 11
    },
    {
      "condition": "direct-astra-web",
      "responses": 96,
      "brandLabels": 288,
      "mentionRate": 0.6076388888888888,
      "accountedUsd": 32.261562500000004,
      "costBasis": [
        "conservative_usage_estimate"
      ],
      "medianLatencySeconds": 28.0,
      "cappedPendingAttemptResponses": 71,
      "moreThanTwoTraceEntriesResponses": 71,
      "parsedSearchCountRange": [
        2,
        2
      ],
      "repeatPairs": 48,
      "differentPanelBrandSetPairs": 10
    },
    {
      "condition": "direct-luna-none",
      "responses": 96,
      "brandLabels": 288,
      "mentionRate": 0.7673611111111112,
      "accountedUsd": 0.109483,
      "costBasis": [
        "conservative_usage_estimate"
      ],
      "medianLatencySeconds": 12.0,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 0,
      "parsedSearchCountRange": [
        0,
        0
      ],
      "repeatPairs": 48,
      "differentPanelBrandSetPairs": 10
    },
    {
      "condition": "direct-luna-web",
      "responses": 96,
      "brandLabels": 288,
      "mentionRate": 0.6979166666666666,
      "accountedUsd": 2.4985099,
      "costBasis": [
        "conservative_usage_estimate"
      ],
      "medianLatencySeconds": 18.0,
      "cappedPendingAttemptResponses": 43,
      "moreThanTwoTraceEntriesResponses": 43,
      "parsedSearchCountRange": [
        1,
        2
      ],
      "repeatPairs": 48,
      "differentPanelBrandSetPairs": 19
    },
    {
      "condition": "direct-sol-none",
      "responses": 96,
      "brandLabels": 288,
      "mentionRate": 0.7604166666666666,
      "accountedUsd": 1.87978,
      "costBasis": [
        "conservative_usage_estimate"
      ],
      "medianLatencySeconds": 22.0,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 0,
      "parsedSearchCountRange": [
        0,
        0
      ],
      "repeatPairs": 48,
      "differentPanelBrandSetPairs": 12
    },
    {
      "condition": "direct-sol-web",
      "responses": 96,
      "brandLabels": 288,
      "mentionRate": 0.6805555555555556,
      "accountedUsd": 14.742265,
      "costBasis": [
        "conservative_usage_estimate"
      ],
      "medianLatencySeconds": 34.5,
      "cappedPendingAttemptResponses": 83,
      "moreThanTwoTraceEntriesResponses": 83,
      "parsedSearchCountRange": [
        1,
        2
      ],
      "repeatPairs": 48,
      "differentPanelBrandSetPairs": 14
    },
    {
      "condition": "openrouter-astra-none",
      "responses": 24,
      "brandLabels": 72,
      "mentionRate": 0.611111111111111,
      "accountedUsd": 0.87476,
      "costBasis": [
        "provider_reported"
      ],
      "medianLatencySeconds": 34.5,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 0,
      "parsedSearchCountRange": [
        0,
        0
      ],
      "repeatPairs": 12,
      "differentPanelBrandSetPairs": 2
    },
    {
      "condition": "openrouter-astra-web",
      "responses": 24,
      "brandLabels": 72,
      "mentionRate": 0.6527777777777778,
      "accountedUsd": 5.900408,
      "costBasis": [
        "provider_reported"
      ],
      "medianLatencySeconds": 44.0,
      "cappedPendingAttemptResponses": 5,
      "moreThanTwoTraceEntriesResponses": 20,
      "parsedSearchCountRange": [
        2,
        2
      ],
      "repeatPairs": 12,
      "differentPanelBrandSetPairs": 2
    },
    {
      "condition": "openrouter-luna-none",
      "responses": 24,
      "brandLabels": 72,
      "mentionRate": 0.736111111111111,
      "accountedUsd": 0.0259556,
      "costBasis": [
        "provider_reported"
      ],
      "medianLatencySeconds": 26.0,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 0,
      "parsedSearchCountRange": [
        0,
        0
      ],
      "repeatPairs": 12,
      "differentPanelBrandSetPairs": 3
    },
    {
      "condition": "openrouter-luna-web",
      "responses": 24,
      "brandLabels": 72,
      "mentionRate": 0.6805555555555556,
      "accountedUsd": 0.55083332,
      "costBasis": [
        "provider_reported"
      ],
      "medianLatencySeconds": 30.5,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 10,
      "parsedSearchCountRange": [
        1,
        2
      ],
      "repeatPairs": 12,
      "differentPanelBrandSetPairs": 2
    },
    {
      "condition": "openrouter-sol-none",
      "responses": 24,
      "brandLabels": 72,
      "mentionRate": 0.736111111111111,
      "accountedUsd": 0.257482,
      "costBasis": [
        "provider_reported"
      ],
      "medianLatencySeconds": 33.0,
      "cappedPendingAttemptResponses": 0,
      "moreThanTwoTraceEntriesResponses": 0,
      "parsedSearchCountRange": [
        0,
        0
      ],
      "repeatPairs": 12,
      "differentPanelBrandSetPairs": 3
    },
    {
      "condition": "openrouter-sol-web",
      "responses": 24,
      "brandLabels": 72,
      "mentionRate": 0.7083333333333333,
      "accountedUsd": 1.6823292,
      "costBasis": [
        "provider_reported"
      ],
      "medianLatencySeconds": 41.5,
      "cappedPendingAttemptResponses": 1,
      "moreThanTwoTraceEntriesResponses": 22,
      "parsedSearchCountRange": [
        2,
        2
      ],
      "repeatPairs": 12,
      "differentPanelBrandSetPairs": 4
    }
  ],
  "categoryNames": [
    "accounting",
    "analytics",
    "automation",
    "crm",
    "email",
    "forms",
    "knowledge",
    "passwords",
    "projects",
    "scheduling",
    "support",
    "uptime"
  ],
  "categoryContrasts": [
    [
      0.0,
      -0.16666666666666674,
      -0.16666666666666663,
      -0.16666666666666663,
      -0.08333333333333326
    ],
    [
      0.0,
      0.02083333333333326,
      -0.25,
      -0.25,
      -0.125
    ],
    [
      -0.02083333333333326,
      0.02083333333333326,
      -0.16666666666666663,
      -0.20833333333333326,
      -0.16666666666666663
    ],
    [
      -0.0625,
      0.02083333333333348,
      -0.04166666666666663,
      -0.16666666666666663,
      -0.04166666666666663
    ],
    [
      0.02083333333333326,
      -0.0625,
      -0.125,
      -0.08333333333333337,
      -0.20833333333333337
    ],
    [
      0.02083333333333326,
      -0.125,
      0.04166666666666663,
      0.16666666666666674,
      0.08333333333333331
    ],
    [
      0.125,
      -0.22916666666666669,
      -0.04166666666666663,
      -0.04166666666666674,
      0.0
    ],
    [
      -0.02083333333333337,
      -0.20833333333333337,
      0.0,
      -0.125,
      -0.125
    ],
    [
      -0.125,
      0.0,
      -0.16666666666666663,
      -0.16666666666666663,
      -0.08333333333333337
    ],
    [
      -0.04166666666666663,
      -0.1250000000000001,
      -0.125,
      -0.04166666666666663,
      -0.12499999999999994
    ],
    [
      -0.0625,
      0.02083333333333337,
      0.0,
      -0.125,
      0.0
    ],
    [
      0.02083333333333337,
      -0.08333333333333337,
      0.20833333333333326,
      0.25,
      0.0
    ]
  ],
  "leaveOneCategoryOut": [
    {
      "contrast": "sol_minus_luna",
      "minimumLeaveOneOutMean": -0.024621212121212124,
      "maximumLeaveOneOutMean": -0.0018939393939393972
    },
    {
      "contrast": "astra_minus_sol",
      "minimumLeaveOneOutMean": -0.08522727272727276,
      "maximumLeaveOneOutMean": -0.06250000000000001
    },
    {
      "contrast": "search_luna",
      "minimumLeaveOneOutMean": -0.09469696969696968,
      "maximumLeaveOneOutMean": -0.053030303030303025
    },
    {
      "contrast": "search_sol",
      "minimumLeaveOneOutMean": -0.10984848484848482,
      "maximumLeaveOneOutMean": -0.06439393939393938
    },
    {
      "contrast": "search_astra",
      "minimumLeaveOneOutMean": -0.08712121212121211,
      "maximumLeaveOneOutMean": -0.060606060606060594
    }
  ],
  "powerFirstQualifyingCandidates": [
    {
      "categories": 36,
      "questions": 144,
      "responses": 1728,
      "sdInflation": 1.0,
      "effect": 0.1,
      "powerByContrast": {
        "sol_minus_luna": 1.0,
        "astra_minus_sol": 1.0,
        "search_luna": 0.997,
        "search_sol": 0.977,
        "search_astra": 1.0
      },
      "minimumPower": 0.977,
      "empiricalNullFamilyWiseError": 0.0595,
      "monteCarloSEAt80Percent": 0.00894427190999916,
      "linearCostScenarioUsd": 165.17385120000003
    },
    {
      "categories": 60,
      "questions": 240,
      "responses": 2880,
      "sdInflation": 1.5,
      "effect": 0.1,
      "powerByContrast": {
        "sol_minus_luna": 1.0,
        "astra_minus_sol": 0.9985,
        "search_luna": 0.962,
        "search_sol": 0.86,
        "search_astra": 1.0
      },
      "minimumPower": 0.86,
      "empiricalNullFamilyWiseError": 0.0375,
      "monteCarloSEAt80Percent": 0.00894427190999916,
      "linearCostScenarioUsd": 275.289752
    },
    {
      "categories": 144,
      "questions": 576,
      "responses": 6912,
      "sdInflation": 2.0,
      "effect": 0.1,
      "powerByContrast": {
        "sol_minus_luna": 1.0,
        "astra_minus_sol": 1.0,
        "search_luna": 0.994,
        "search_sol": 0.9575,
        "search_astra": 1.0
      },
      "minimumPower": 0.9575,
      "empiricalNullFamilyWiseError": 0.0455,
      "monteCarloSEAt80Percent": 0.00894427190999916,
      "linearCostScenarioUsd": 660.6954048000001
    }
  ],
  "limitations": [
    "Automated pilot labels; see the dated audit record for AI review scope. No human adjudication.",
    "Leave-one-category-out means are post-hoc descriptive sensitivity, not new hypothesis tests.",
    "Search counters and trace status normalization differ across routes; execution parity is unverified.",
    "Linear future costs assume the observed mix and token usage; they are not price quotes or admitted spend."
  ],
  "scoringRevision": "2026-09-16 explicit final-answer Markdown links included; primary labels unchanged",
  "secondaryCitationExploratory": [
    {
      "contrast": "sol_minus_luna",
      "difference": 0.0069444444444444545,
      "ci95": [
        -0.022569444444444437,
        0.036458333333333356
      ],
      "p": 0.6662093556260982,
      "holmP": 1.0,
      "categories": 12
    },
    {
      "contrast": "astra_minus_sol",
      "difference": -0.01041666666666668,
      "ci95": [
        -0.05902777777777779,
        0.03819444444444444
      ],
      "p": 0.6987040988884521,
      "holmP": 1.0,
      "categories": 12
    },
    {
      "contrast": "search_luna",
      "difference": 0.673611111111111,
      "ci95": [
        0.5833333333333333,
        0.763888888888889
      ],
      "p": 3.1063845529892085e-08,
      "holmP": 9.319153658967626e-08,
      "categories": 12
    },
    {
      "contrast": "search_sol",
      "difference": 0.6666666666666666,
      "ci95": [
        0.59375,
        0.7395833333333334
      ],
      "p": 2.906362403428167e-09,
      "holmP": 1.4531812017140835e-08,
      "categories": 12
    },
    {
      "contrast": "search_astra",
      "difference": 0.5416666666666666,
      "ci95": [
        0.4791666666666667,
        0.6006944444444445
      ],
      "p": 4.179022669495232e-09,
      "holmP": 1.6716090677980928e-08,
      "categories": 12
    }
  ],
  "conditionSummaries": [
    {
      "condition": "direct-luna-none",
      "plannedResponses": 96,
      "validResponses": 96,
      "mentionedMissingOutcomeBounds": [
        0.7673611111111112,
        0.7673611111111112
      ],
      "citedMissingOutcomeBounds": [
        0.0,
        0.0
      ]
    },
    {
      "condition": "direct-luna-web",
      "plannedResponses": 96,
      "validResponses": 96,
      "mentionedMissingOutcomeBounds": [
        0.6979166666666666,
        0.6979166666666666
      ],
      "citedMissingOutcomeBounds": [
        0.673611111111111,
        0.673611111111111
      ]
    },
    {
      "condition": "direct-sol-none",
      "plannedResponses": 96,
      "validResponses": 96,
      "mentionedMissingOutcomeBounds": [
        0.7604166666666666,
        0.7604166666666666
      ],
      "citedMissingOutcomeBounds": [
        0.010416666666666666,
        0.010416666666666666
      ]
    },
    {
      "condition": "direct-sol-web",
      "plannedResponses": 96,
      "validResponses": 96,
      "mentionedMissingOutcomeBounds": [
        0.6805555555555555,
        0.6805555555555555
      ],
      "citedMissingOutcomeBounds": [
        0.6770833333333334,
        0.6770833333333334
      ]
    },
    {
      "condition": "direct-astra-none",
      "plannedResponses": 96,
      "validResponses": 96,
      "mentionedMissingOutcomeBounds": [
        0.6805555555555555,
        0.6805555555555555
      ],
      "citedMissingOutcomeBounds": [
        0.0625,
        0.0625
      ]
    },
    {
      "condition": "direct-astra-web",
      "plannedResponses": 96,
      "validResponses": 96,
      "mentionedMissingOutcomeBounds": [
        0.607638888888889,
        0.607638888888889
      ],
      "citedMissingOutcomeBounds": [
        0.6041666666666667,
        0.6041666666666667
      ]
    },
    {
      "condition": "openrouter-luna-none",
      "plannedResponses": 24,
      "validResponses": 24,
      "mentionedMissingOutcomeBounds": [
        0.736111111111111,
        0.736111111111111
      ],
      "citedMissingOutcomeBounds": [
        0.0,
        0.0
      ]
    },
    {
      "condition": "openrouter-luna-web",
      "plannedResponses": 24,
      "validResponses": 24,
      "mentionedMissingOutcomeBounds": [
        0.6805555555555555,
        0.6805555555555555
      ],
      "citedMissingOutcomeBounds": [
        0.6805555555555555,
        0.6805555555555555
      ]
    },
    {
      "condition": "openrouter-sol-none",
      "plannedResponses": 24,
      "validResponses": 24,
      "mentionedMissingOutcomeBounds": [
        0.736111111111111,
        0.736111111111111
      ],
      "citedMissingOutcomeBounds": [
        0.0,
        0.0
      ]
    },
    {
      "condition": "openrouter-sol-web",
      "plannedResponses": 24,
      "validResponses": 24,
      "mentionedMissingOutcomeBounds": [
        0.7083333333333334,
        0.7083333333333334
      ],
      "citedMissingOutcomeBounds": [
        0.6666666666666666,
        0.6666666666666666
      ]
    },
    {
      "condition": "openrouter-astra-none",
      "plannedResponses": 24,
      "validResponses": 24,
      "mentionedMissingOutcomeBounds": [
        0.611111111111111,
        0.611111111111111
      ],
      "citedMissingOutcomeBounds": [
        0.09722222222222221,
        0.09722222222222221
      ]
    },
    {
      "condition": "openrouter-astra-web",
      "plannedResponses": 24,
      "validResponses": 24,
      "mentionedMissingOutcomeBounds": [
        0.6527777777777778,
        0.6527777777777778
      ],
      "citedMissingOutcomeBounds": [
        0.6527777777777778,
        0.6527777777777778
      ]
    }
  ],
  "review": {
    "reviewedDate": "2026-09-16",
    "type": "AI-assisted audit, not human validation",
    "independent": false,
    "blinded": false,
    "sampleResponses": 150,
    "sampleBrandLabels": 450,
    "samplePositiveContextsInspected": 319,
    "sampleNegativeAliasChecks": 131,
    "primaryMentionCorrections": 0,
    "sampleCitationCorrections": 8,
    "fullCohortCitationCorrections": 28,
    "fullCohortAffectedResponses": 13,
    "originalCitedLabels": 707,
    "correctedCitedLabels": 735,
    "citationSupport": "not assessed",
    "urlAccessibility": "not assessed",
    "interRaterAgreement": "not applicable; one AI-assisted audit",
    "priorAmbiguousPositiveAudit": {
      "labels": 298,
      "corrections": 0,
      "overlapsSample": true
    }
  },
  "cost": {
    "accountedUsd": 65.21720342,
    "excludedCapabilityChecksUsd": 0.8674849,
    "ceilingUsd": 100,
    "invoiceReconciled": false,
    "basis": "Conservative direct estimates plus provider-reported gateway costs; excludes subscriptions and labor"
  },
  "rawResponsesReleased": false
}