mcloh/decision-models-benchmark
1
0
Fork 0
mirror of https://github.com/mcloh/decision-models-benchmark.git synced 2026-11-05 22:04:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Calibração e congelamento da configuração (tarefas 60–62)

Browse Source 
- config/calibration.json: temperatura e limiar de abstenção por candidato e
  dispositivo, ajustados só na partição de calibração.
- config/frozen.json: hashes de configuração, calibração, partições,
  adaptadores, contrato e ambientes; libera a partição de teste.
- Q6: lote de produção = 1; tarefa 65 não se aplica neste ciclo (D4).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Marcos Lohmann
2026-10-07 18:54:20 -03:00
parent 8e5e78b841
commit d4564d9c1d
19 changed files with 7338 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
config/benchmark.yaml
View File

@@ -51,8 +51,7 @@ input: # Q8
execution:
seed: 20261007
batch_sweep: [1, 2, 4, 8, 16, 32] # Q6
production_batch: auto # maior lote com p95 <= slo
production_batch: 1 # Q6: contrato por requisição, decisão turno a turno
repetitions: 3
cpu: # Q11
timeout_ms: 10000
@@ -70,7 +69,6 @@ calibration: # Q10
criteria: # Q5
latency_gpu_batch1:
p95_ms_per_decision: 150
p95_ms_per_turn: 300
accuracy:
tasks: [D1, D2]
non_inferiority_margin_pp: 3.0

118
config/calibration.json Normal file
View File

@@ -0,0 +1,118 @@
{
"method": "temperature_scaling",
"fit_split": "calibration",
"max_confident_wrong_rate": 0.02,
"min_coverage": 0.8,
"candidates": {
"gliner": {
"cpu": {
"temperature": 0.8187,
"fitted_temperature": 0.8187,
"accepted": true,
"n": 1127,
"ece_raw": 0.0767,
"ece_calibrated": 0.0631,
"ece_cv_raw": 0.1016,
"ece_cv_calibrated": 0.0872,
"nll_raw": 1.3205,
"nll_calibrated": 1.305,
"threshold": 0.856167,
"coverage": 0.118,
"selective_accuracy": 0.8346,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
},
"cuda": {
"temperature": 0.8187,
"fitted_temperature": 0.8187,
"accepted": true,
"n": 1127,
"ece_raw": 0.0767,
"ece_calibrated": 0.0634,
"ece_cv_raw": 0.1016,
"ece_cv_calibrated": 0.0872,
"nll_raw": 1.3205,
"nll_calibrated": 1.305,
"threshold": 0.857453,
"coverage": 0.118,
"selective_accuracy": 0.8346,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
}
},
"laya": {
"cpu": {
"temperature": 3.5787,
"fitted_temperature": 3.5787,
"accepted": true,
"n": 1127,
"ece_raw": 0.3296,
"ece_calibrated": 0.0715,
"ece_cv_raw": 0.3321,
"ece_cv_calibrated": 0.1171,
"nll_raw": 2.8006,
"nll_calibrated": 1.5575,
"threshold": 0.871532,
"coverage": 0.0515,
"selective_accuracy": 0.8448,
"confident_wrong_rate": 0.008,
"meets_min_coverage": false
},
"cuda": {
"temperature": 3.5787,
"fitted_temperature": 3.5787,
"accepted": true,
"n": 1127,
"ece_raw": 0.3263,
"ece_calibrated": 0.075,
"ece_cv_raw": 0.3318,
"ece_cv_calibrated": 0.113,
"nll_raw": 2.7748,
"nll_calibrated": 1.5517,
"threshold": 0.871532,
"coverage": 0.0515,
"selective_accuracy": 0.8448,
"confident_wrong_rate": 0.008,
"meets_min_coverage": false
}
},
"rizzo_flow": {
"cuda": {
"temperature": 1.2523,
"fitted_temperature": 1.2523,
"accepted": true,
"n": 1127,
"ece_raw": 0.0577,
"ece_calibrated": 0.0206,
"ece_cv_raw": 0.0825,
"ece_cv_calibrated": 0.0584,
"nll_raw": 0.5447,
"nll_calibrated": 0.52,
"threshold": 0.923792,
"coverage": 0.4765,
"selective_accuracy": 0.959,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
}
},
"semif": {
"cuda": {
"temperature": 1.284,
"fitted_temperature": 1.284,
"accepted": true,
"n": 1127,
"ece_raw": 0.0552,
"ece_calibrated": 0.0225,
"ece_cv_raw": 0.0699,
"ece_cv_calibrated": 0.061,
"nll_raw": 0.4776,
"nll_calibrated": 0.4545,
"threshold": 0.914887,
"coverage": 0.5954,
"selective_accuracy": 0.9672,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
}
}
}
}

72
config/frozen.json Normal file
View File

@@ -0,0 +1,72 @@
{
"frozen_at": "2026-10-07T21:53:53+00:00",
"files": {
"config/benchmark.yaml": "53546bced660a618206023f3054ab478b0cba390369b4915c60374a4ff59b8f6",
"config/calibration.json": "2918aa77d2251f60d155d3f0be8271d5b72c0945412e270344359bb1e8f7fa28",
"data/splits/MANIFEST.json": "50377ab93dbaa71aa3220b722c2270fb76ddda4f5a24763ba4ad3623a2e704ac",
"env/artifacts-manifest.json": "2cde61d21cec5086a221af988bea5dfb9a31fb4f3269511e63a23ca2b2bbfb6c",
"contracts/jev/schemas/request.schema.json": "e44e5af9798b96453b676c05b216015fd2b62c9590301adce8c7807989774d35",
"contracts/jev/schemas/response.schema.json": "008f8c2e68064b46b0f0649e654326ffd65a4eb8c080e9d23a6be4ebc44cd752",
"harness/evaluate.py": "2c696e1da8c5e06284c73adf93b842cf7e541135e78dabd9cd1751825ec2866a",
"dmb/canonical.py": "833e92490c2a34b9469e8e9aa4ee3aea6f2411ad74165a8df129ce460c2fd8d7",
"dmb/jev.py": "eb9b97bc84d65d6481b3f86ea6f242c445bde02bfb0518bed735a15b802e66c9",
"dmb/server.py": "cb8a7515337de781c90918dc2a40e499a44d6db52e6a78ae31f98fa22d415249",
"dmb/adapter.py": "361e25b990a0de345fa79663f1593acf0bc62ef9580d6dfec1e35d4942a2839d",
"adapters/laya/adapter.py": "b332c57330f24a08334b794cd933e267dfd5f82b1b073bf6213abb42e1d6d6e8",
"adapters/semif/adapter.py": "83039c3e07dc2d64e809c3f33f3187a1e2fa3d8e5b20b204652dd673a354da3c",
"adapters/gliner/adapter.py": "e499caea077353ccdc6cf458e8cb5fde6f34ef7149d463726e6b5d199f435921",
"adapters/rizzo_flow/adapter.py": "8621b36f1b970ab778ff4af6c5e7cba3a933a0b2bfe6c6861f49dfd7c819d79b"
},
"environments": {
"laya": {
"candidate": "laya",
"object": "envs/laya/20261007T151753Z.tar.gz",
"sha256": "2003611792956150ce85ecc9ea7635f9b1f8dc7280c5da507e3a303188ed11aa",
"bytes": 4308288985,
"built_at": "20261007T151753Z",
"code_rev": "5d93903",
"python": "Python 3.11.9",
"home": "/home/datascience"
},
"gliner": {
"candidate": "gliner",
"object": "envs/gliner/20261007T151810Z.tar.gz",
"sha256": "fff384c86db233995f5a9182768bef0a8f27052436a3531f455027bde1d19954",
"bytes": 4314977274,
"built_at": "20261007T151810Z",
"code_rev": "5d93903",
"python": "Python 3.11.9",
"home": "/home/datascience"
},
"semif": {
"candidate": "semif",
"object": "envs/semif/20261007T151340Z.tar.gz",
"sha256": "50b090041721d0faa626ad44ebbcfa9b5c4b37ee9f6f69da181757580a4f5065",
"bytes": 4312946412,
"built_at": "20261007T151340Z",
"code_rev": "5d93903",
"python": "Python 3.11.9",
"home": "/home/datascience"
},
"rizzo_flow": {
"candidate": "rizzo_flow",
"object": "envs/rizzo_flow/20261007T151336Z.tar.gz",
"sha256": "805376a152a984010ca838e4134a9410f9c6e12ab97a0095297f7d7ea79b61c4",
"bytes": 693207602,
"built_at": "20261007T151336Z",
"code_rev": "5d93903",
"python": "Python 3.11.9",
"home": "/home/datascience"
},
"rizzo_flow_cuda_runtime": {
"object": "runtimes/rizzo_flow/llama-b11081-linux-x64-cuda-dsjobs-20261007T165909Z.tar.gz",
"sha256": "08e03dc5ca1e64f6fd73b5a52cf0eb72727f203a15fe3bf86f79526d25ae00a9",
"dir": "llama-b11081-linux-x64-cuda-dsjobs",
"llama_commit": "161755f29e415e2c33efe906e91843c068efd664",
"cuda": "12.8",
"archs": "80;86",
"built_at": "20261007T165909Z",
"code_rev": "f322ed06"
}
}
}