{
  "schema": "aiyvo.usage_report",
  "schema_version": "1.6",
  "data_status": "SYNTHETIC",
  "provider_verified_billing": "absent",
  "as_of": "2026-10-11T00:00:00+00:00",
  "input": {
    "format": "csv",
    "row_kind": "request",
    "row_kind_source": "mapping",
    "data_source": "synthetic",
    "cost_basis": "not_declared",
    "zero_costs": "not_declared",
    "records": 672,
    "currency": "USD",
    "period": {
      "start": "2026-09-28T00:00:00+00:00",
      "end": "2026-10-04T23:45:00+00:00"
    },
    "dropped_columns": [],
    "warnings": []
  },
  "provenance": {
    "note": "SHA-256 fingerprints identify the exact files and analyzer source used, so the report can be reproduced. They prove nothing about whether the data is true or complete: the data is not independently verified, and the fingerprints are not a signature or a custody record.",
    "status": "complete",
    "analyzer": {
      "name": "AIYVO offline usage analyzer",
      "report_schema": "1.6",
      "source_sha256": "dc759f963d31da373750dfd674f9ddd0824fffe2bbd29e531460b907e2dfcf70"
    },
    "input": {
      "sha256": "809be4b64ba5651786634621983900e884ce922c675594e9f6865a2dde171a60",
      "format": "csv"
    },
    "mapping": {
      "sha256": "7e3152d4614714c543c4c165324fc0680d070ecea41ce5672f4d229aebe92551",
      "detail": null
    },
    "price_table": {
      "sha256": "0e70df3c41b6fecc5ead10fde57ed91908d16545dc954b556ca47befb5fb3b67",
      "detail": "basis synthetic"
    },
    "reference_time": "2026-10-11T00:00:00+00:00",
    "reference_time_source": "--as-of",
    "data_source": "synthetic",
    "data_source_basis": "the input is AIYVO's bundled synthetic example"
  },
  "price_table": {
    "basis": "synthetic",
    "currency": "USD",
    "entries": 5,
    "status": "not verified; not necessarily current"
  },
  "value_summary": {
    "title": "The value AIYVO uncovered",
    "data_status": "synthetic",
    "currency": "USD",
    "headline": {
      "total_spend_analyzed": {
        "label": "Total AI spend analyzed",
        "value": "50.003705",
        "display": "$50.00",
        "status": "observed",
        "calculation": "sum of reported_cost over the 668 of 672 requests that have a cost; 4 requests without a cost are not included",
        "sources": [
          "input file (SHA-256 under Provenance), reported_cost column (synthetic)"
        ]
      },
      "potential_savings": {
        "label": "Potential savings identified",
        "value": null,
        "display": "Not yet quantified",
        "status": "not_yet_quantified",
        "calculation": "The groups in this file served different requests, and quality and prices are not verified, so moving work between them has no measured price. A saving needs the same work measured on both options, or confirmed prices and volumes; this analysis has neither, so AIYVO estimates no saving.",
        "sources": []
      },
      "spend_requiring_review": {
        "label": "Spend requiring further review",
        "value": "10.730104",
        "display": "$10.73",
        "status": "observed",
        "calculation": "sum of reported_cost over the 297 requests with a cost that failed, have no known outcome, or are in a group too small to compare; each request is counted once",
        "sources": [
          "input file (SHA-256 under Provenance), reported_cost column (synthetic)",
          "input file, outcome column",
          "report.json groups[*].eligible_for_comparison"
        ],
        "note": "Investigation spend, not recoverable savings",
        "meaning": "Investigation spend, not recoverable savings: money already spent on requests that are worth looking into. AIYVO does not estimate how much of it, if any, could be recovered."
      },
      "evidence_backed_findings": {
        "label": "Evidence-backed findings",
        "value": 5,
        "display": "5",
        "status": "observed",
        "calculation": "findings below, each with its figures, calculation and sources",
        "sources": []
      }
    },
    "realized_savings": {
      "label": "Verified realized savings",
      "value": null,
      "display": "None verified",
      "status": "none_verified",
      "calculation": "A realized saving needs provider billing from before and after a change; none was supplied, and AIYVO has made no change to your usage.",
      "sources": []
    },
    "findings": [
      {
        "id": "price_per_request_misleads",
        "category": "observed",
        "title": "The lowest price per request is not the lowest cost per successful request",
        "significance": "Judged on price per request, provider_c/model-c1 has the lowest figure. Counting what failures cost, provider_b/model-b1 has the lowest cost per successful request ($0.0684 against $0.1474). A choice made on price per request alone would miss this. The groups served different requests, so this is an observation, not a recommendation.",
        "figures": [
          {
            "label": "provider_c/model-c1: reported cost per request",
            "value": "0.014744",
            "display": "$0.0147",
            "status": "observed",
            "calculation": "reported cost of provider_c/model-c1's requests with a cost \u00f7 those requests",
            "sources": [
              "report.json groups[provider_c/model-c1].metrics.reported_cost_per_record",
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)"
            ]
          },
          {
            "label": "provider_b/model-b1: cost per successful request",
            "value": "0.068425",
            "display": "$0.0684",
            "status": "observed",
            "calculation": "reported cost of all provider_b/model-b1's requests, failures included \u00f7 its successful requests",
            "sources": [
              "report.json groups[provider_b/model-b1].metrics.cost_per_success",
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)"
            ]
          },
          {
            "label": "provider_c/model-c1: cost per successful request",
            "value": "0.147445",
            "display": "$0.1474",
            "status": "observed",
            "calculation": "reported cost of all provider_c/model-c1's requests, failures included \u00f7 its successful requests",
            "sources": [
              "report.json groups[provider_c/model-c1].metrics.cost_per_success",
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)"
            ]
          }
        ]
      },
      {
        "id": "spend_on_failures",
        "category": "observed",
        "title": "21.0% of reported spend went to requests that failed",
        "significance": "$10.49 of $50.00 paid for requests whose outcome was failure. Failures are paid for, so they belong in what AI work really costs; why they failed is worth reviewing.",
        "figures": [
          {
            "label": "Reported cost of failed requests",
            "value": "10.488737",
            "display": "$10.49",
            "status": "observed",
            "calculation": "sum of reported_cost over the 289 requests with outcome failure and a cost",
            "sources": [
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)",
              "input file, outcome column"
            ]
          }
        ]
      },
      {
        "id": "spend_with_thin_evidence",
        "category": "observed",
        "title": "Spend AIYVO could not assess",
        "significance": "$0.34 is in 1 group with too few requests to compare (provider_d/model-d1). AIYVO draws no conclusion from it.",
        "figures": [
          {
            "label": "Reported cost in groups withheld from comparison",
            "value": "0.336149",
            "display": "$0.34",
            "status": "observed",
            "calculation": "sum of reported_cost over the 12 requests with a cost in provider_d/model-d1",
            "sources": [
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)",
              "report.json groups[provider_d/model-d1].evidence.exclusions"
            ]
          }
        ]
      },
      {
        "id": "spend_not_reported",
        "category": "observed",
        "title": "4 requests with no reported cost",
        "significance": "Their spend is not in any total and is not estimated (provider_e/model-e1). Complete cost data would make the comparison stronger.",
        "figures": [
          {
            "label": "Spend of requests without a reported cost",
            "value": null,
            "display": "Not yet quantified",
            "status": "not_yet_quantified",
            "calculation": "4 requests have an empty cost; AIYVO never fills in a missing cost",
            "sources": [
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)"
            ]
          }
        ]
      },
      {
        "id": "reported_above_list_estimate",
        "category": "estimated",
        "title": "Reported cost above the list-price estimate",
        "significance": "$0.08 more was reported than the supplied list prices would give for the same requests (provider_e/model-e1: 5.0% above). Worth checking against the provider's prices and invoices. This is an estimate: the price table (synthetic) is not verified, and the difference is not a saving.",
        "figures": [
          {
            "label": "provider_e/model-e1: reported cost minus list-price estimate",
            "value": "0.078965",
            "display": "$0.08",
            "status": "estimated",
            "calculation": "sum over the 56 of 60 requests with a cost of (reported_cost \u2212 list-price estimate from token counts \u00d7 the price table's rates)",
            "sources": [
              "input file (SHA-256 under Provenance), reported_cost column (synthetic)",
              "input file, token columns",
              "price table (basis synthetic, SHA-256 under Provenance, not verified)",
              "report.json groups[provider_e/model-e1].metrics.reported_vs_list_estimate"
            ]
          }
        ]
      }
    ],
    "categories": {
      "observed": "taken directly from the reported data, as reported",
      "estimated": "depends on a supplied price table that AIYVO has not verified",
      "verified_realized": "none: no realized saving has been verified"
    }
  },
  "metric_definitions": {
    "record": "one row of the input file",
    "row_kind": "request; declared in the mapping file",
    "success": "a record whose outcome is success: a successful request",
    "success_rate": "request success rate: successful requests / (requests with outcome success or failure)",
    "cost_per_success": "cost per successful request: reported cost of all requests, failures included / number of successful requests",
    "task_level": "withheld: rows are requests and AIYVO does not group requests into tasks, so cost per completed task cannot be derived from this file"
  },
  "rules": {
    "coverage_threshold": "0.90",
    "cost_per_success_requires": "100% cost and outcome coverage",
    "min_sample_per_group": 30,
    "min_successes_to_rank": 10,
    "min_successes_to_rank_status": "provisional rule; not statistically validated",
    "tie_places": 4,
    "ties": "values equal when rounded half-up to 4 decimal places share a rank and are marked tied; values are reported at 6 decimal places",
    "zero_cost": "a reported 0 is shown as reported; cost orderings that include a group reporting 0 are withheld unless the mapping declares zero_costs \"genuine\"; an empty cost is missing, never 0, and is never replaced by a list-price estimate"
  },
  "groups": [
    {
      "group": "provider_a/model-a1",
      "provider": "provider_a",
      "model": "model-a1",
      "eligible_for_comparison": true,
      "metrics": {
        "records": {
          "value": "200",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "input_tokens": {
          "value": "8081449",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "output_tokens": {
          "value": "1005600",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "cached_input_tokens": {
          "value": "1653068",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_cost_total": {
          "value": "34.865075",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_cost_per_record": {
          "value": "0.174325",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "list_price_estimate_total": {
          "value": "34.865063",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_vs_list_estimate": {
          "value": "0.000000",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "success_rate": {
          "value": "0.865000",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "outcome_counts": {
          "success": 173,
          "failure": 27,
          "unknown": 0,
          "missing": 0
        },
        "cost_per_success": {
          "value": "0.201532",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "latency_ms_median": {
          "value": "25190.5",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "latency_ms_p90": {
          "value": "33178",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "quality_score_mean": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 0,
            "total": 200
          },
          "withheld_reason": "not present in the data"
        }
      },
      "evidence": {
        "observations": 200,
        "observation_period": {
          "start": "2026-09-28T00:00:00+00:00",
          "end": "2026-10-04T23:45:00+00:00"
        },
        "outcomes": {
          "success": 173,
          "failure": 27,
          "unknown": 0,
          "missing": 0
        },
        "cost_coverage": {
          "covered": 200,
          "total": 200
        },
        "cost_status": {
          "reported_nonzero": 200,
          "reported_zero": 0,
          "missing": 0,
          "basis": "not_declared",
          "basis_detail": "basis not declared",
          "zero_interpretation": "not_declared",
          "zero_interpretation_detail": "not declared: a reported 0 may be a real cost of 0 or an unknown cost exported as 0",
          "missing_treatment": "missing, never zero: not estimated, not filled in, not counted as free",
          "list_price_estimate": "an AIYVO estimate, shown separately; never used in place of a reported cost",
          "provider_verified": 0
        },
        "outcome_coverage": {
          "covered": 200,
          "total": 200
        },
        "task_types": {
          "refactor": 100,
          "bugfix": 100
        },
        "source": {
          "reported_cost": "synthetic",
          "outcome": "synthetic",
          "latency": "synthetic",
          "list_price_estimate": "aiyvo_estimate"
        },
        "bases": {
          "cost_per_success": {
            "unit": "cost per successful request",
            "numerator": "reported cost of all requests, failures included",
            "numerator_value": "34.865075",
            "denominator": "successful requests",
            "denominator_value": 173
          },
          "success_rate": {
            "unit": "request success rate",
            "numerator": "successful requests",
            "numerator_value": 173,
            "denominator": "requests with outcome success or failure",
            "denominator_value": 200
          },
          "reported_cost_per_record": {
            "unit": "reported cost per request",
            "numerator": "reported cost of the requests that have a cost",
            "numerator_value": "34.865075",
            "denominator": "requests with a reported cost",
            "denominator_value": 200
          }
        },
        "missing_data": "Nothing is imputed. Records without a cost lower cost coverage, are never counted as free, and are left out of cost averages; the cost total is withheld unless every record has a cost. Unknown and missing outcomes are left out of the success rate. The cost per successful request needs every record to have both a cost and a known outcome.",
        "eligibility": "eligible",
        "exclusions": []
      }
    },
    {
      "group": "provider_b/model-b1",
      "provider": "provider_b",
      "model": "model-b1",
      "eligible_for_comparison": true,
      "metrics": {
        "records": {
          "value": "200",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "input_tokens": {
          "value": "7642206",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "output_tokens": {
          "value": "993168",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "cached_input_tokens": {
          "value": "1422185",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_cost_total": {
          "value": "10.195276",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_cost_per_record": {
          "value": "0.050976",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "list_price_estimate_total": {
          "value": "11.328080",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_vs_list_estimate": {
          "value": "-0.100000",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "success_rate": {
          "value": "0.745000",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "outcome_counts": {
          "success": 149,
          "failure": 51,
          "unknown": 0,
          "missing": 0
        },
        "cost_per_success": {
          "value": "0.068425",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "latency_ms_median": {
          "value": "15090.5",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "latency_ms_p90": {
          "value": "18910",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "quality_score_mean": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 0,
            "total": 200
          },
          "withheld_reason": "not present in the data"
        }
      },
      "evidence": {
        "observations": 200,
        "observation_period": {
          "start": "2026-09-28T00:45:00+00:00",
          "end": "2026-10-04T23:30:00+00:00"
        },
        "outcomes": {
          "success": 149,
          "failure": 51,
          "unknown": 0,
          "missing": 0
        },
        "cost_coverage": {
          "covered": 200,
          "total": 200
        },
        "cost_status": {
          "reported_nonzero": 200,
          "reported_zero": 0,
          "missing": 0,
          "basis": "not_declared",
          "basis_detail": "basis not declared",
          "zero_interpretation": "not_declared",
          "zero_interpretation_detail": "not declared: a reported 0 may be a real cost of 0 or an unknown cost exported as 0",
          "missing_treatment": "missing, never zero: not estimated, not filled in, not counted as free",
          "list_price_estimate": "an AIYVO estimate, shown separately; never used in place of a reported cost",
          "provider_verified": 0
        },
        "outcome_coverage": {
          "covered": 200,
          "total": 200
        },
        "task_types": {
          "refactor": 100,
          "bugfix": 100
        },
        "source": {
          "reported_cost": "synthetic",
          "outcome": "synthetic",
          "latency": "synthetic",
          "list_price_estimate": "aiyvo_estimate"
        },
        "bases": {
          "cost_per_success": {
            "unit": "cost per successful request",
            "numerator": "reported cost of all requests, failures included",
            "numerator_value": "10.195276",
            "denominator": "successful requests",
            "denominator_value": 149
          },
          "success_rate": {
            "unit": "request success rate",
            "numerator": "successful requests",
            "numerator_value": 149,
            "denominator": "requests with outcome success or failure",
            "denominator_value": 200
          },
          "reported_cost_per_record": {
            "unit": "reported cost per request",
            "numerator": "reported cost of the requests that have a cost",
            "numerator_value": "10.195276",
            "denominator": "requests with a reported cost",
            "denominator_value": 200
          }
        },
        "missing_data": "Nothing is imputed. Records without a cost lower cost coverage, are never counted as free, and are left out of cost averages; the cost total is withheld unless every record has a cost. Unknown and missing outcomes are left out of the success rate. The cost per successful request needs every record to have both a cost and a known outcome.",
        "eligibility": "eligible",
        "exclusions": []
      }
    },
    {
      "group": "provider_c/model-c1",
      "provider": "provider_c",
      "model": "model-c1",
      "eligible_for_comparison": true,
      "metrics": {
        "records": {
          "value": "200",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "input_tokens": {
          "value": "8086033",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "output_tokens": {
          "value": "1032827",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "cached_input_tokens": {
          "value": "1616187",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_cost_total": {
          "value": "2.948897",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_cost_per_record": {
          "value": "0.014744",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "list_price_estimate_total": {
          "value": "2.948900",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "reported_vs_list_estimate": {
          "value": "-0.000001",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "success_rate": {
          "value": "0.100000",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "outcome_counts": {
          "success": 20,
          "failure": 180,
          "unknown": 0,
          "missing": 0
        },
        "cost_per_success": {
          "value": "0.147445",
          "label": "synthetic",
          "coverage": {
            "covered": 200,
            "total": 200
          },
          "withheld_reason": null
        },
        "latency_ms_median": {
          "value": "6924.5",
          "label": "synthetic",
          "coverage": {
            "covered": 190,
            "total": 200
          },
          "withheld_reason": null
        },
        "latency_ms_p90": {
          "value": "9394",
          "label": "synthetic",
          "coverage": {
            "covered": 190,
            "total": 200
          },
          "withheld_reason": null
        },
        "quality_score_mean": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 0,
            "total": 200
          },
          "withheld_reason": "not present in the data"
        }
      },
      "evidence": {
        "observations": 200,
        "observation_period": {
          "start": "2026-09-28T01:15:00+00:00",
          "end": "2026-10-04T21:00:00+00:00"
        },
        "outcomes": {
          "success": 20,
          "failure": 180,
          "unknown": 0,
          "missing": 0
        },
        "cost_coverage": {
          "covered": 200,
          "total": 200
        },
        "cost_status": {
          "reported_nonzero": 200,
          "reported_zero": 0,
          "missing": 0,
          "basis": "not_declared",
          "basis_detail": "basis not declared",
          "zero_interpretation": "not_declared",
          "zero_interpretation_detail": "not declared: a reported 0 may be a real cost of 0 or an unknown cost exported as 0",
          "missing_treatment": "missing, never zero: not estimated, not filled in, not counted as free",
          "list_price_estimate": "an AIYVO estimate, shown separately; never used in place of a reported cost",
          "provider_verified": 0
        },
        "outcome_coverage": {
          "covered": 200,
          "total": 200
        },
        "task_types": {
          "refactor": 100,
          "bugfix": 100
        },
        "source": {
          "reported_cost": "synthetic",
          "outcome": "synthetic",
          "latency": "synthetic",
          "list_price_estimate": "aiyvo_estimate"
        },
        "bases": {
          "cost_per_success": {
            "unit": "cost per successful request",
            "numerator": "reported cost of all requests, failures included",
            "numerator_value": "2.948897",
            "denominator": "successful requests",
            "denominator_value": 20
          },
          "success_rate": {
            "unit": "request success rate",
            "numerator": "successful requests",
            "numerator_value": 20,
            "denominator": "requests with outcome success or failure",
            "denominator_value": 200
          },
          "reported_cost_per_record": {
            "unit": "reported cost per request",
            "numerator": "reported cost of the requests that have a cost",
            "numerator_value": "2.948897",
            "denominator": "requests with a reported cost",
            "denominator_value": 200
          }
        },
        "missing_data": "Nothing is imputed. Records without a cost lower cost coverage, are never counted as free, and are left out of cost averages; the cost total is withheld unless every record has a cost. Unknown and missing outcomes are left out of the success rate. The cost per successful request needs every record to have both a cost and a known outcome.",
        "eligibility": "eligible",
        "exclusions": []
      }
    },
    {
      "group": "provider_d/model-d1",
      "provider": "provider_d",
      "model": "model-d1",
      "eligible_for_comparison": false,
      "metrics": {
        "records": {
          "value": "12",
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": null
        },
        "input_tokens": {
          "value": "524283",
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": null
        },
        "output_tokens": {
          "value": "52483",
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": null
        },
        "cached_input_tokens": {
          "value": "127112",
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": null
        },
        "reported_cost_total": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "reported_cost_per_record": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "list_price_estimate_total": {
          "value": null,
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "reported_vs_list_estimate": {
          "value": null,
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "success_rate": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "outcome_counts": {
          "success": 8,
          "failure": 4,
          "unknown": 0,
          "missing": 0
        },
        "cost_per_success": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "latency_ms_median": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "latency_ms_p90": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 12,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        },
        "quality_score_mean": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 0,
            "total": 12
          },
          "withheld_reason": "insufficient sample: 12 valid records; at least 30 are required"
        }
      },
      "evidence": {
        "observations": 12,
        "observation_period": {
          "start": "2026-09-28T18:15:00+00:00",
          "end": "2026-10-04T16:45:00+00:00"
        },
        "outcomes": {
          "success": 8,
          "failure": 4,
          "unknown": 0,
          "missing": 0
        },
        "cost_coverage": {
          "covered": 12,
          "total": 12
        },
        "cost_status": {
          "reported_nonzero": 12,
          "reported_zero": 0,
          "missing": 0,
          "basis": "not_declared",
          "basis_detail": "basis not declared",
          "zero_interpretation": "not_declared",
          "zero_interpretation_detail": "not declared: a reported 0 may be a real cost of 0 or an unknown cost exported as 0",
          "missing_treatment": "missing, never zero: not estimated, not filled in, not counted as free",
          "list_price_estimate": "an AIYVO estimate, shown separately; never used in place of a reported cost",
          "provider_verified": 0
        },
        "outcome_coverage": {
          "covered": 12,
          "total": 12
        },
        "task_types": {
          "refactor": 6,
          "bugfix": 6
        },
        "source": {
          "reported_cost": "synthetic",
          "outcome": "synthetic",
          "latency": "synthetic",
          "list_price_estimate": "aiyvo_estimate"
        },
        "bases": {
          "cost_per_success": {
            "unit": "cost per successful request",
            "numerator": "reported cost of all requests, failures included",
            "numerator_value": null,
            "denominator": "successful requests",
            "denominator_value": null
          },
          "success_rate": {
            "unit": "request success rate",
            "numerator": "successful requests",
            "numerator_value": 8,
            "denominator": "requests with outcome success or failure",
            "denominator_value": 12
          },
          "reported_cost_per_record": {
            "unit": "reported cost per request",
            "numerator": "reported cost of the requests that have a cost",
            "numerator_value": "0.336149",
            "denominator": "requests with a reported cost",
            "denominator_value": 12
          }
        },
        "missing_data": "Nothing is imputed. Records without a cost lower cost coverage, are never counted as free, and are left out of cost averages; the cost total is withheld unless every record has a cost. Unknown and missing outcomes are left out of the success rate. The cost per successful request needs every record to have both a cost and a known outcome.",
        "eligibility": "withheld",
        "exclusions": [
          {
            "category": "insufficient_sample",
            "label": "Not enough observations",
            "condition": "failed",
            "detail": "insufficient sample: 12 valid records; at least 30 are required",
            "remediation": "Add valid records until this group has at least 30; it has 12. Then re-run the analysis."
          }
        ]
      }
    },
    {
      "group": "provider_e/model-e1",
      "provider": "provider_e",
      "model": "model-e1",
      "eligible_for_comparison": true,
      "metrics": {
        "records": {
          "value": "60",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "input_tokens": {
          "value": "2339312",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "output_tokens": {
          "value": "300446",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "cached_input_tokens": {
          "value": "428797",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "reported_cost_total": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 56,
            "total": 60
          },
          "withheld_reason": "cost missing on 4 of 60 records; a total would count them as free"
        },
        "reported_cost_per_record": {
          "value": "0.029613",
          "label": "synthetic",
          "coverage": {
            "covered": 56,
            "total": 60
          },
          "withheld_reason": null
        },
        "list_price_estimate_total": {
          "value": "1.727812",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "reported_vs_list_estimate": {
          "value": "0.049998",
          "label": "aiyvo_estimate",
          "coverage": {
            "covered": 56,
            "total": 60
          },
          "withheld_reason": null
        },
        "success_rate": {
          "value": "0.516667",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "outcome_counts": {
          "success": 31,
          "failure": 29,
          "unknown": 0,
          "missing": 0
        },
        "cost_per_success": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 56,
            "total": 60
          },
          "withheld_reason": "requires 100% cost and outcome coverage; cost present in 56 of 60 records (93.3%), success or failure known in 60 of 60 records (100.0%)"
        },
        "latency_ms_median": {
          "value": "11619.5",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "latency_ms_p90": {
          "value": "15219",
          "label": "synthetic",
          "coverage": {
            "covered": 60,
            "total": 60
          },
          "withheld_reason": null
        },
        "quality_score_mean": {
          "value": null,
          "label": "synthetic",
          "coverage": {
            "covered": 0,
            "total": 60
          },
          "withheld_reason": "not present in the data"
        }
      },
      "evidence": {
        "observations": 60,
        "observation_period": {
          "start": "2026-09-28T02:30:00+00:00",
          "end": "2026-10-04T23:00:00+00:00"
        },
        "outcomes": {
          "success": 31,
          "failure": 29,
          "unknown": 0,
          "missing": 0
        },
        "cost_coverage": {
          "covered": 56,
          "total": 60
        },
        "cost_status": {
          "reported_nonzero": 56,
          "reported_zero": 0,
          "missing": 4,
          "basis": "not_declared",
          "basis_detail": "basis not declared",
          "zero_interpretation": "not_declared",
          "zero_interpretation_detail": "not declared: a reported 0 may be a real cost of 0 or an unknown cost exported as 0",
          "missing_treatment": "missing, never zero: not estimated, not filled in, not counted as free",
          "list_price_estimate": "an AIYVO estimate, shown separately; never used in place of a reported cost",
          "provider_verified": 0
        },
        "outcome_coverage": {
          "covered": 60,
          "total": 60
        },
        "task_types": {
          "refactor": 30,
          "bugfix": 30
        },
        "source": {
          "reported_cost": "synthetic",
          "outcome": "synthetic",
          "latency": "synthetic",
          "list_price_estimate": "aiyvo_estimate"
        },
        "bases": {
          "cost_per_success": {
            "unit": "cost per successful request",
            "numerator": "reported cost of all requests, failures included",
            "numerator_value": null,
            "denominator": "successful requests",
            "denominator_value": null
          },
          "success_rate": {
            "unit": "request success rate",
            "numerator": "successful requests",
            "numerator_value": 31,
            "denominator": "requests with outcome success or failure",
            "denominator_value": 60
          },
          "reported_cost_per_record": {
            "unit": "reported cost per request",
            "numerator": "reported cost of the requests that have a cost",
            "numerator_value": "1.658308",
            "denominator": "requests with a reported cost",
            "denominator_value": 56
          }
        },
        "missing_data": "Nothing is imputed. Records without a cost lower cost coverage, are never counted as free, and are left out of cost averages; the cost total is withheld unless every record has a cost. Unknown and missing outcomes are left out of the success rate. The cost per successful request needs every record to have both a cost and a known outcome.",
        "eligibility": "eligible",
        "exclusions": [
          {
            "category": "cost_coverage",
            "label": "Cost data incomplete",
            "condition": "unknown",
            "detail": "cost present in 56 of 60 records (93.3%)",
            "remediation": "Supply a reported cost for every record (0 where a record was free), then re-run. Records without a cost are not estimated or filled in."
          }
        ]
      }
    }
  ],
  "comparison": {
    "comparability": {
      "basis": "same_task_mix",
      "detail": "every compared group has the same share of each task type"
    },
    "rankings_withheld": null,
    "cost_rankings_withheld": [],
    "ranked_by_cost_per_success": [
      {
        "rank": 1,
        "group": "provider_b/model-b1",
        "cost_per_success": "0.068425",
        "successes": 149,
        "records": 200,
        "tied": false
      },
      {
        "rank": 2,
        "group": "provider_c/model-c1",
        "cost_per_success": "0.147445",
        "successes": 20,
        "records": 200,
        "tied": false
      },
      {
        "rank": 3,
        "group": "provider_a/model-a1",
        "cost_per_success": "0.201532",
        "successes": 173,
        "records": 200,
        "tied": false
      }
    ],
    "ranked_by_reported_cost_per_record": [
      {
        "rank": 1,
        "group": "provider_c/model-c1",
        "reported_cost_per_record": "0.014744",
        "cost_coverage": {
          "covered": 200,
          "total": 200
        },
        "tied": false
      },
      {
        "rank": 2,
        "group": "provider_e/model-e1",
        "reported_cost_per_record": "0.029613",
        "cost_coverage": {
          "covered": 56,
          "total": 60
        },
        "tied": false
      },
      {
        "rank": 3,
        "group": "provider_b/model-b1",
        "reported_cost_per_record": "0.050976",
        "cost_coverage": {
          "covered": 200,
          "total": 200
        },
        "tied": false
      },
      {
        "rank": 4,
        "group": "provider_a/model-a1",
        "reported_cost_per_record": "0.174325",
        "cost_coverage": {
          "covered": 200,
          "total": 200
        },
        "tied": false
      }
    ],
    "not_ranked": [],
    "note": "Observational ordering of eligible groups only; not a recommendation and not a controlled benchmark. cost_per_success is cost per successful request (see metric_definitions). Values equal at 4 decimal places share a rank."
  },
  "limitations": [
    "All usage, cost, outcome and latency values are synthetic (fictitious example data), not customer data, and were not independently verified.",
    "No provider-verified billing was used. Reported cost may differ from what a provider invoices.",
    "List-price estimates are AIYVO estimates from the supplied price table; those prices are not verified and not necessarily current.",
    "Comparisons are observational: different groups served different requests. This is not a controlled experiment.",
    "Token counts from different models are not directly comparable (different tokenizers).",
    "No savings, quality superiority or recommendation is claimed.",
    "Rows are requests, so a success is a successful request, not a completed task. AIYVO does not group requests into tasks: when a task takes several requests (retries, agent steps), cost per successful request can be far below the cost of a completed task, and no per-task figure is given.",
    "The file mixes task types. The compared groups have the same mix, but results for one task type can differ from the mix; use --by-task-type to see each task type separately."
  ]
}