{
  "evaluationVersion": "offline-synthetic-v1",
  "measuredAt": "2026-07-15T17:51:18.000Z",
  "fixtureCount": 20,
  "providerMode": "mock",
  "liveCalls": 0,
  "metrics": {
    "structuredOutputValidity": {
      "label": "Structured-output validity",
      "target": "100%",
      "numerator": 20,
      "denominator": 20,
      "percent": 100,
      "passed": true
    },
    "deterministicArithmeticCorrectness": {
      "label": "Deterministic arithmetic correctness",
      "target": "100%",
      "numerator": 20,
      "denominator": 20,
      "percent": 100,
      "passed": true
    },
    "criticalFieldExtractionAccuracy": {
      "label": "Critical-field extraction accuracy",
      "target": ">=90%",
      "numerator": 331,
      "denominator": 332,
      "percent": 99.7,
      "passed": true
    },
    "findingCategoryPrecision": {
      "label": "Finding-category precision",
      "target": ">=90%",
      "numerator": 38,
      "denominator": 39,
      "percent": 97.44,
      "passed": true
    },
    "findingCategoryRecall": {
      "label": "Finding-category recall",
      "target": ">=90%",
      "numerator": 38,
      "denominator": 38,
      "percent": 100,
      "passed": true
    },
    "sourceReferenceResolvability": {
      "label": "Source-reference resolvability",
      "target": "100%",
      "numerator": 20,
      "denominator": 20,
      "percent": 100,
      "passed": true
    },
    "clarificationQuestionsTiedToFindings": {
      "label": "Clarification questions tied to findings",
      "target": "100%",
      "numerator": 20,
      "denominator": 20,
      "percent": 100,
      "passed": true
    },
    "fixtureOutcomeAgreement": {
      "label": "End-to-end fixture outcome agreement",
      "target": ">=90%",
      "numerator": 19,
      "denominator": 20,
      "percent": 95,
      "passed": true
    }
  },
  "fixtureResults": [
    {
      "id": "canonical-initial",
      "title": "Canonical initial discrepancy",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 82,
        "status": "review_required",
        "unresolvedConditionCount": 3
      },
      "observedOutcome": {
        "coverageScore": 82,
        "status": "review_required",
        "unresolvedConditionCount": 3
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-unexplained-total:unexplained",
        "finding-missing-apr:missing",
        "finding-missing-term:missing",
        "finding-missing-protection-optionality:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-unexplained-total:unexplained",
        "finding-missing-apr:missing",
        "finding-missing-term:missing",
        "finding-missing-protection-optionality:missing"
      ]
    },
    {
      "id": "canonical-revised",
      "title": "Canonical dealer response",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 93,
        "status": "review_required",
        "unresolvedConditionCount": 2
      },
      "observedOutcome": {
        "coverageScore": 93,
        "status": "review_required",
        "unresolvedConditionCount": 2
      },
      "expectedRiskFindingPairs": [
        "finding-changed-monthly-payment:changed",
        "finding-conditional-payment:conditional",
        "finding-missing-apr:missing",
        "finding-missing-service-contract-optionality:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-changed-monthly-payment:changed",
        "finding-conditional-payment:conditional",
        "finding-missing-apr:missing",
        "finding-missing-service-contract-optionality:missing"
      ]
    },
    {
      "id": "complete-quote",
      "title": "Complete reconciled quote",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": ["finding-conditional-payment:conditional"]
    },
    {
      "id": "missing-apr",
      "title": "APR absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 98,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 98,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-apr:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-apr:missing"
      ]
    },
    {
      "id": "mock-misses-apr",
      "title": "Known extraction miss",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": false,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 98,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-apr:missing"
      ],
      "knownFailure": "The offline mock extraction omits a source-stated APR, so the downstream trail correctly exposes a missing-APR finding but does not match source ground truth."
    },
    {
      "id": "missing-term",
      "title": "Finance term absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 98,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 98,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-term:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-term:missing"
      ]
    },
    {
      "id": "missing-product-optionality",
      "title": "Protection-package optionality absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 95,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 95,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-protection-optionality:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-protection-optionality:missing"
      ]
    },
    {
      "id": "unexplained-gap",
      "title": "Unexplained stated-total gap",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 91,
        "status": "review_required",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 91,
        "status": "review_required",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-unexplained-total:unexplained"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-unexplained-total:unexplained"
      ]
    },
    {
      "id": "missing-stated-total",
      "title": "Stated total absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 86,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 86,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-stated-total:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-stated-total:missing"
      ]
    },
    {
      "id": "missing-vehicle",
      "title": "Vehicle identity absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 90,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 90,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-missing-vehicle:missing",
        "finding-conditional-payment:conditional"
      ],
      "observedRiskFindingPairs": [
        "finding-missing-vehicle:missing",
        "finding-conditional-payment:conditional"
      ]
    },
    {
      "id": "missing-selling-price",
      "title": "Selling price absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 86,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 86,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-missing-price:missing",
        "finding-conditional-payment:conditional"
      ],
      "observedRiskFindingPairs": [
        "finding-missing-price:missing",
        "finding-conditional-payment:conditional"
      ]
    },
    {
      "id": "missing-incentive-eligibility",
      "title": "Incentive eligibility absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 95,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 95,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-incentive-eligibility:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-incentive-eligibility:missing"
      ]
    },
    {
      "id": "missing-cash-separation",
      "title": "Cash-price separation absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 97,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 97,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-cash-separation:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-cash-separation:missing"
      ]
    },
    {
      "id": "missing-trade-separation",
      "title": "Trade-in separation absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 98,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 98,
        "status": "not_comparable_yet",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-trade-separation:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-trade-separation:missing"
      ]
    },
    {
      "id": "no-dealer-fee",
      "title": "No dealer fee",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 91,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 91,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": ["finding-conditional-payment:conditional"]
    },
    {
      "id": "no-dealer-product",
      "title": "No dealer-installed product",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 90,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 90,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": ["finding-conditional-payment:conditional"]
    },
    {
      "id": "optional-service-contract",
      "title": "Optional service contract",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": ["finding-conditional-payment:conditional"]
    },
    {
      "id": "service-contract-optionality-absent",
      "title": "Service-contract optionality absent",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 95,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "observedOutcome": {
        "coverageScore": 95,
        "status": "review_required",
        "unresolvedConditionCount": 1
      },
      "expectedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-service-contract-optionality:missing"
      ],
      "observedRiskFindingPairs": [
        "finding-conditional-payment:conditional",
        "finding-missing-service-contract-optionality:missing"
      ]
    },
    {
      "id": "trade-allowance-discount",
      "title": "Trade allowance sign",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": ["finding-conditional-payment:conditional"]
    },
    {
      "id": "decimal-cent-arithmetic",
      "title": "Cent-precise arithmetic",
      "structuredOutputValid": true,
      "arithmeticCorrect": true,
      "outcomeAgreement": true,
      "sourceReferencesResolvable": true,
      "clarificationQuestionsTied": true,
      "expectedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "observedOutcome": {
        "coverageScore": 100,
        "status": "comparable_as_documented",
        "unresolvedConditionCount": 0
      },
      "expectedRiskFindingPairs": ["finding-conditional-payment:conditional"],
      "observedRiskFindingPairs": ["finding-conditional-payment:conditional"]
    }
  ],
  "knownFailures": [
    "mock-misses-apr: The offline mock extraction omits a source-stated APR, so the downstream trail correctly exposes a missing-APR finding but does not match source ground truth."
  ],
  "limitations": [
    "The offline scores measure committed synthetic sources and mock outputs; they are not a claim about live-model or OCR accuracy.",
    "One fixture deliberately omits a source-stated APR from its mock extraction so metric failure propagation remains visible.",
    "PDF and image fixtures validate evidence contracts and deterministic behavior, not visual OCR quality.",
    "TermsTrail displays documented tax amounts but does not independently calculate tax rates or provide legal or financial advice."
  ],
  "liveEvaluation": {
    "status": "not_run_protected",
    "reason": "Paid live evaluation remains disabled until explicitly authorized with credentials and a call or spend cap."
  }
}