mcloh/poc-grok-websearch
1
0
Fork 0
mirror of https://github.com/mcloh/poc-grok-websearch.git synced 2026-11-05 22:04:12 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
poc-grok-websearch/benchmarks/results/comparison-2026-10-15/summary.json

262 lines
8.0 KiB
JSON
Raw Permalink Blame History

{
"run_id": "comparison-2026-10-15",
"status": "complete",
"method": "100 prompts idênticos por modelo, OCI real, sem histórico e sem retry; concorrência 2",
"region": "us-chicago-1",
"reference_date": "2026-10-15",
"manifest": {
"dataset_sha256": "cd019da35e180ef111479f14b328fac0a7a0c5d76edfb493083d52378683742b",
"prices_sha256": "2c92b5befe11f8481e6a8d5f6453e7036e6df36f73d0aaeed4ceee9d27aaa536",
"system_sha256": "8a6fa58dbc2539af95cf138848caf4ac704abb6ee738ff197df2859f19788b42",
"models": [
"xai.grok-4.20-non-reasoning",
"xai.grok-4.3"
],
"reference_date": "2026-10-15",
"region": "us-chicago-1",
"concurrency": 2,
"max_output_tokens": 2400,
"retries": 0,
"transport_timeout_seconds": 150
},
"total": {
"interactions": 200,
"http_200": 200,
"websearch_confirmed": 200,
"unknown_cost": 0,
"total_usd": "11.24303815",
"mean_usd": "0.05621519075",
"median_usd": "0.05276945",
"p95_usd": "0.09127515",
"web_search_usd": "4.795",
"input_usd": "5.42838625",
"cached_input_usd": "0.5010944",
"output_usd": "0.5185575",
"input_tokens": 6848181,
"cached_input_tokens": 2505472,
"output_tokens": 207423,
"reasoning_tokens": 69002,
"web_search_calls": 959,
"median_seconds": 12.1395
},
"by_model": {
"xai.grok-4.20-non-reasoning": {
"interactions": 100,
"http_200": 100,
"websearch_confirmed": 100,
"unknown_cost": 0,
"total_usd": "7.17872765",
"mean_usd": "0.0717872765",
"median_usd": "0.06975135",
"p95_usd": "0.0944971",
"web_search_usd": "3.235",
"input_usd": "3.27227625",
"cached_input_usd": "0.4395264",
"output_usd": "0.2319250",
"input_tokens": 4815453,
"cached_input_tokens": 2197632,
"output_tokens": 92770,
"reasoning_tokens": 0,
"web_search_calls": 647,
"median_seconds": 13.5515
},
"xai.grok-4.3": {
"interactions": 100,
"http_200": 100,
"websearch_confirmed": 100,
"unknown_cost": 0,
"total_usd": "4.06431050",
"mean_usd": "0.040643105",
"median_usd": "0.0389775",
"p95_usd": "0.05617225",
"web_search_usd": "1.560",
"input_usd": "2.15611000",
"cached_input_usd": "0.0615680",
"output_usd": "0.2866325",
"input_tokens": 2032728,
"cached_input_tokens": 307840,
"output_tokens": 114653,
"reasoning_tokens": 69002,
"web_search_calls": 312,
"median_seconds": 10.594999999999999
}
},
"by_category": {
"culinaria": {
"xai.grok-4.20-non-reasoning": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "1.67610895",
"mean_usd": "0.067044358",
"median_usd": "0.06645185",
"p95_usd": "0.09347575",
"web_search_usd": "0.800",
"input_usd": "0.72552625",
"cached_input_usd": "0.0874752",
"output_usd": "0.0631075",
"input_tokens": 1017797,
"cached_input_tokens": 437376,
"output_tokens": 25243,
"reasoning_tokens": 0,
"web_search_calls": 160,
"median_seconds": 14.484
},
"xai.grok-4.3": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "1.05350780",
"mean_usd": "0.042140312",
"median_usd": "0.0410104",
"p95_usd": "0.0583708",
"web_search_usd": "0.410",
"input_usd": "0.55128000",
"cached_input_usd": "0.0195328",
"output_usd": "0.0726950",
"input_tokens": 538688,
"cached_input_tokens": 97664,
"output_tokens": 29078,
"reasoning_tokens": 16979,
"web_search_calls": 82,
"median_seconds": 11.07
}
},
"esportes": {
"xai.grok-4.20-non-reasoning": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "1.88192095",
"mean_usd": "0.075276838",
"median_usd": "0.07266885",
"p95_usd": "0.0944971",
"web_search_usd": "0.790",
"input_usd": "0.91507875",
"cached_input_usd": "0.1244672",
"output_usd": "0.0523750",
"input_tokens": 1354399,
"cached_input_tokens": 622336,
"output_tokens": 20950,
"reasoning_tokens": 0,
"web_search_calls": 158,
"median_seconds": 12.815
},
"xai.grok-4.3": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "0.97694465",
"mean_usd": "0.039077786",
"median_usd": "0.0392477",
"p95_usd": "0.0466627",
"web_search_usd": "0.355",
"input_usd": "0.53925875",
"cached_input_usd": "0.0125184",
"output_usd": "0.0701675",
"input_tokens": 493999,
"cached_input_tokens": 62592,
"output_tokens": 28067,
"reasoning_tokens": 16887,
"web_search_calls": 71,
"median_seconds": 10.272
}
},
"clima": {
"xai.grok-4.20-non-reasoning": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "1.81859655",
"mean_usd": "0.072743862",
"median_usd": "0.0693353",
"p95_usd": "0.09727325",
"web_search_usd": "0.820",
"input_usd": "0.81494625",
"cached_input_usd": "0.1296128",
"output_usd": "0.0540375",
"input_tokens": 1300021,
"cached_input_tokens": 648064,
"output_tokens": 21615,
"reasoning_tokens": 0,
"web_search_calls": 164,
"median_seconds": 13.31
},
"xai.grok-4.3": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "0.96629490",
"mean_usd": "0.038651796",
"median_usd": "0.03733645",
"p95_usd": "0.04932415",
"web_search_usd": "0.385",
"input_usd": "0.49851000",
"cached_input_usd": "0.0148224",
"output_usd": "0.0679625",
"input_tokens": 472920,
"cached_input_tokens": 74112,
"output_tokens": 27185,
"reasoning_tokens": 17081,
"web_search_calls": 77,
"median_seconds": 10.563
}
},
"entretenimento": {
"xai.grok-4.20-non-reasoning": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "1.80210120",
"mean_usd": "0.072084048",
"median_usd": "0.07050875",
"p95_usd": "0.09117215",
"web_search_usd": "0.825",
"input_usd": "0.81672500",
"cached_input_usd": "0.0979712",
"output_usd": "0.0624050",
"input_tokens": 1143236,
"cached_input_tokens": 489856,
"output_tokens": 24962,
"reasoning_tokens": 0,
"web_search_calls": 165,
"median_seconds": 14.406
},
"xai.grok-4.3": {
"interactions": 25,
"http_200": 25,
"websearch_confirmed": 25,
"unknown_cost": 0,
"total_usd": "1.06756315",
"mean_usd": "0.042702526",
"median_usd": "0.0390671",
"p95_usd": "0.05719645",
"web_search_usd": "0.410",
"input_usd": "0.56706125",
"cached_input_usd": "0.0146944",
"output_usd": "0.0758075",
"input_tokens": 527121,
"cached_input_tokens": 73472,
"output_tokens": 30323,
"reasoning_tokens": 18055,
"web_search_calls": 82,
"median_seconds": 10.673
}
}
},
"paired_comparisons": 100,
"paired_cheaper_counts": {
"xai.grok-4.3": 95,
"xai.grok-4.20-non-reasoning": 5
},
"billing_note": "Custo calculado de tokens e buscas medidos, a preços de lista Oracle; não é fatura conciliada. Sem impostos, descontos, VM, rede ou armazenamento.",
"quality_note": "Confirmação de busca não prova correção factual, completude ou superioridade da resposta."
}
Reference in New Issue View Git Blame Copy Permalink