mcloh/decision-models-benchmark
1
0
Fork 0
mirror of https://github.com/mcloh/decision-models-benchmark.git synced 2026-11-05 22:04:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Calibração e congelamento da configuração (tarefas 60–62)

Browse Source 
- config/calibration.json: temperatura e limiar de abstenção por candidato e
  dispositivo, ajustados só na partição de calibração.
- config/frozen.json: hashes de configuração, calibração, partições,
  adaptadores, contrato e ambientes; libera a partição de teste.
- Q6: lote de produção = 1; tarefa 65 não se aplica neste ciclo (D4).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Marcos Lohmann
2026-10-07 18:54:20 -03:00
parent 8e5e78b841
commit d4564d9c1d
19 changed files with 7338 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
config/calibration.json Normal file
View File

@@ -0,0 +1,118 @@
{
"method": "temperature_scaling",
"fit_split": "calibration",
"max_confident_wrong_rate": 0.02,
"min_coverage": 0.8,
"candidates": {
"gliner": {
"cpu": {
"temperature": 0.8187,
"fitted_temperature": 0.8187,
"accepted": true,
"n": 1127,
"ece_raw": 0.0767,
"ece_calibrated": 0.0631,
"ece_cv_raw": 0.1016,
"ece_cv_calibrated": 0.0872,
"nll_raw": 1.3205,
"nll_calibrated": 1.305,
"threshold": 0.856167,
"coverage": 0.118,
"selective_accuracy": 0.8346,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
},
"cuda": {
"temperature": 0.8187,
"fitted_temperature": 0.8187,
"accepted": true,
"n": 1127,
"ece_raw": 0.0767,
"ece_calibrated": 0.0634,
"ece_cv_raw": 0.1016,
"ece_cv_calibrated": 0.0872,
"nll_raw": 1.3205,
"nll_calibrated": 1.305,
"threshold": 0.857453,
"coverage": 0.118,
"selective_accuracy": 0.8346,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
}
},
"laya": {
"cpu": {
"temperature": 3.5787,
"fitted_temperature": 3.5787,
"accepted": true,
"n": 1127,
"ece_raw": 0.3296,
"ece_calibrated": 0.0715,
"ece_cv_raw": 0.3321,
"ece_cv_calibrated": 0.1171,
"nll_raw": 2.8006,
"nll_calibrated": 1.5575,
"threshold": 0.871532,
"coverage": 0.0515,
"selective_accuracy": 0.8448,
"confident_wrong_rate": 0.008,
"meets_min_coverage": false
},
"cuda": {
"temperature": 3.5787,
"fitted_temperature": 3.5787,
"accepted": true,
"n": 1127,
"ece_raw": 0.3263,
"ece_calibrated": 0.075,
"ece_cv_raw": 0.3318,
"ece_cv_calibrated": 0.113,
"nll_raw": 2.7748,
"nll_calibrated": 1.5517,
"threshold": 0.871532,
"coverage": 0.0515,
"selective_accuracy": 0.8448,
"confident_wrong_rate": 0.008,
"meets_min_coverage": false
}
},
"rizzo_flow": {
"cuda": {
"temperature": 1.2523,
"fitted_temperature": 1.2523,
"accepted": true,
"n": 1127,
"ece_raw": 0.0577,
"ece_calibrated": 0.0206,
"ece_cv_raw": 0.0825,
"ece_cv_calibrated": 0.0584,
"nll_raw": 0.5447,
"nll_calibrated": 0.52,
"threshold": 0.923792,
"coverage": 0.4765,
"selective_accuracy": 0.959,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
}
},
"semif": {
"cuda": {
"temperature": 1.284,
"fitted_temperature": 1.284,
"accepted": true,
"n": 1127,
"ece_raw": 0.0552,
"ece_calibrated": 0.0225,
"ece_cv_raw": 0.0699,
"ece_cv_calibrated": 0.061,
"nll_raw": 0.4776,
"nll_calibrated": 0.4545,
"threshold": 0.914887,
"coverage": 0.5954,
"selective_accuracy": 0.9672,
"confident_wrong_rate": 0.0195,
"meets_min_coverage": false
}
}
}
}