mcloh/decision-models-benchmark
1
0
Fork 0
mirror of https://github.com/mcloh/decision-models-benchmark.git synced 2026-11-05 22:04:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

setup models

Browse Source
This commit is contained in:
Marcos Lohmann
2026-10-07 13:43:09 -03:00
parent aeee6c7de1
commit 84ea9a5307
75 changed files with 5093 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
reports/sanity/gliner-cpu.json Normal file
View File

@@ -0,0 +1,219 @@
{
"candidate": {
"name": "gliner",
"model_id": "gliner@a35a0cd3b7a0",
"revision": "a35a0cd3b7a0f00f2effc576f454cd48fa98aa5f",
"adapter_version": "0.1.0",
"device": "cpu",
"context_limit": 4096,
"probabilities_source": "native"
},
"device": "cpu",
"host": "host-local",
"load_seconds": 15.93,
"conformance": [
{
"name": "valid_choice_noul",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_score",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_state_object",
"expected": "ok",
"got": "ok"
},
{
"name": "invalid_17_options",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_single_option",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_noul_missing_false",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_unknown_type",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_empty_state",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_no_questions",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_extra_field",
"expected": "invalid_request",
"got": "invalid_request"
}
],
"sanity": [
{
"name": "choice:pagamento",
"gold": "pagamento",
"pred": "pagamento",
"latency_ms": 303.8,
"answer": {
"type": "choice",
"choice": "pagamento",
"confidence": 0.6344146988487307,
"probabilities": {
"pagamento": 0.6344146988487307,
"cancelamento": 0.016834578235374244,
"suporte_tecnico": 0.022163211344910134,
"sem_correspondencia": 0.326587511570985
}
},
"pred_permuted": "pagamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:cancelamento",
"gold": "cancelamento",
"pred": "cancelamento",
"latency_ms": 297.9,
"answer": {
"type": "choice",
"choice": "cancelamento",
"confidence": 0.8718586499105494,
"probabilities": {
"pagamento": 0.010419030596174071,
"cancelamento": 0.8718586499105494,
"suporte_tecnico": 0.012619386333082717,
"sem_correspondencia": 0.10510293316019376
}
},
"pred_permuted": "cancelamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:suporte_tecnico",
"gold": "suporte_tecnico",
"pred": "sem_correspondencia",
"latency_ms": 218.5,
"answer": {
"type": "choice",
"choice": "sem_correspondencia",
"confidence": 0.6342991318760811,
"probabilities": {
"pagamento": 0.020201073094038888,
"cancelamento": 0.04278706948440038,
"suporte_tecnico": 0.30271272554547957,
"sem_correspondencia": 0.6342991318760811
}
},
"pred_permuted": "sem_correspondencia",
"permutation_invariant": true,
"correct": false
},
{
"name": "choice:sem_correspondencia",
"gold": "sem_correspondencia",
"pred": "sem_correspondencia",
"latency_ms": 226.3,
"answer": {
"type": "choice",
"choice": "sem_correspondencia",
"confidence": 0.860428139738716,
"probabilities": {
"pagamento": 0.028759410242109536,
"cancelamento": 0.03164806754541749,
"suporte_tecnico": 0.07916438247375696,
"sem_correspondencia": 0.860428139738716
}
},
"pred_permuted": "sem_correspondencia",
"permutation_invariant": true,
"correct": true
},
{
"name": "noul:clara",
"gold": "true",
"pred": "false",
"latency_ms": 184.3,
"answer": {
"type": "noul",
"noul": 0.2368283016907438
},
"correct": false
},
{
"name": "noul:vaga",
"gold": "false",
"pred": "false",
"latency_ms": 227.3,
"answer": {
"type": "noul",
"noul": 0.20341360543566903
},
"correct": true
},
{
"name": "score:alta",
"gold": "2",
"pred": "0",
"latency_ms": 189.4,
"answer": {
"type": "score",
"score": 0.9812671227395942,
"confidence": 0.36140790832709446,
"probabilities": {
"0": 0.36140790832709446,
"1": 0.29591706060621686,
"2": 0.3426750310666887
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": false
},
{
"name": "score:baixa",
"gold": "0",
"pred": "0",
"latency_ms": 158.1,
"answer": {
"type": "score",
"score": 0.4816726286216157,
"confidence": 0.6750846907546932,
"probabilities": {
"0": 0.6750846907546932,
"1": 0.16815798986899794,
"2": 0.15675731937630888
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": true
}
],
"summary": {
"contract_violations": [],
"sanity_correct": "5/8",
"permutation_invariant": "4/4"
}
}

219
reports/sanity/laya-cpu.json Normal file
View File

@@ -0,0 +1,219 @@
{
"candidate": {
"name": "laya",
"model_id": "laya@1720e3e3357c",
"revision": "1720e3e3357cfe1e281542e223f8273b0890ca34",
"adapter_version": "0.1.0",
"device": "cpu",
"context_limit": 1024,
"probabilities_source": "native"
},
"device": "cpu",
"host": "host-local",
"load_seconds": 8.4,
"conformance": [
{
"name": "valid_choice_noul",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_score",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_state_object",
"expected": "ok",
"got": "ok"
},
{
"name": "invalid_17_options",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_single_option",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_noul_missing_false",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_unknown_type",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_empty_state",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_no_questions",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_extra_field",
"expected": "invalid_request",
"got": "invalid_request"
}
],
"sanity": [
{
"name": "choice:pagamento",
"gold": "pagamento",
"pred": "pagamento",
"latency_ms": 114.4,
"answer": {
"type": "choice",
"choice": "pagamento",
"confidence": 0.9977,
"probabilities": {
"pagamento": 0.9997979802020598,
"cancelamento": 1.0000980096049412e-06,
"suporte_tecnico": 1.0000980096049412e-06,
"sem_correspondencia": 0.00020001960192098824
}
},
"pred_permuted": "pagamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:cancelamento",
"gold": "cancelamento",
"pred": "cancelamento",
"latency_ms": 137.1,
"answer": {
"type": "choice",
"choice": "cancelamento",
"confidence": 1.0,
"probabilities": {
"pagamento": 9.99997000009e-07,
"cancelamento": 0.999997000009,
"suporte_tecnico": 9.99997000009e-07,
"sem_correspondencia": 9.99997000009e-07
}
},
"pred_permuted": "cancelamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:suporte_tecnico",
"gold": "suporte_tecnico",
"pred": "suporte_tecnico",
"latency_ms": 154.9,
"answer": {
"type": "choice",
"choice": "suporte_tecnico",
"confidence": 0.2935,
"probabilities": {
"pagamento": 0.0496049604960496,
"cancelamento": 0.047204720472047206,
"suporte_tecnico": 0.5706570657065706,
"sem_correspondencia": 0.33253325332533257
}
},
"pred_permuted": "suporte_tecnico",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:sem_correspondencia",
"gold": "sem_correspondencia",
"pred": "sem_correspondencia",
"latency_ms": 115.2,
"answer": {
"type": "choice",
"choice": "sem_correspondencia",
"confidence": 0.7909,
"probabilities": {
"pagamento": 0.0034996500349965005,
"cancelamento": 0.0044995500449955,
"suporte_tecnico": 0.0642935706429357,
"sem_correspondencia": 0.9277072292770723
}
},
"pred_permuted": "sem_correspondencia",
"permutation_invariant": true,
"correct": true
},
{
"name": "noul:clara",
"gold": "true",
"pred": "true",
"latency_ms": 99.9,
"answer": {
"type": "noul",
"noul": 0.9982
},
"correct": true
},
{
"name": "noul:vaga",
"gold": "false",
"pred": "false",
"latency_ms": 88.8,
"answer": {
"type": "noul",
"noul": 0.1563
},
"correct": true
},
{
"name": "score:alta",
"gold": "2",
"pred": "2",
"latency_ms": 97.6,
"answer": {
"type": "score",
"score": 1.1202,
"confidence": 0.01,
"probabilities": {
"0": 0.2718,
"1": 0.3362,
"2": 0.392
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": true
},
{
"name": "score:baixa",
"gold": "0",
"pred": "1",
"latency_ms": 91.0,
"answer": {
"type": "score",
"score": 1.0998999999999999,
"confidence": 0.1514,
"probabilities": {
"0": 0.1477,
"1": 0.6047,
"2": 0.2476
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": false
}
],
"summary": {
"contract_violations": [],
"sanity_correct": "7/8",
"permutation_invariant": "4/4"
}
}

219
reports/sanity/rizzo_flow-cpu.json Normal file
View File

@@ -0,0 +1,219 @@
{
"candidate": {
"name": "rizzo_flow",
"model_id": "rizzo_flow@55633c8cbd2b",
"revision": "55633c8cbd2b826bd3eefdeb05310450996649df",
"adapter_version": "0.1.0",
"device": "cpu",
"context_limit": 2048,
"probabilities_source": "native"
},
"device": "cpu",
"host": "host-local",
"load_seconds": 6.18,
"conformance": [
{
"name": "valid_choice_noul",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_score",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_state_object",
"expected": "ok",
"got": "ok"
},
{
"name": "invalid_17_options",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_single_option",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_noul_missing_false",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_unknown_type",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_empty_state",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_no_questions",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_extra_field",
"expected": "invalid_request",
"got": "invalid_request"
}
],
"sanity": [
{
"name": "choice:pagamento",
"gold": "pagamento",
"pred": "pagamento",
"latency_ms": 7483.9,
"answer": {
"type": "choice",
"choice": "pagamento",
"confidence": 0.9939581449644029,
"probabilities": {
"pagamento": 0.9954686087233023,
"cancelamento": 0.0014294878384867886,
"suporte_tecnico": 0.0006663578744809516,
"sem_correspondencia": 0.0024355455637299515
}
},
"pred_permuted": "pagamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:cancelamento",
"gold": "cancelamento",
"pred": "cancelamento",
"latency_ms": 7733.7,
"answer": {
"type": "choice",
"choice": "cancelamento",
"confidence": 0.9959894668208951,
"probabilities": {
"pagamento": 0.0006611097629735292,
"cancelamento": 0.9969921001156713,
"suporte_tecnico": 0.0005694308917635679,
"sem_correspondencia": 0.0017773592295916517
}
},
"pred_permuted": "cancelamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:suporte_tecnico",
"gold": "suporte_tecnico",
"pred": "suporte_tecnico",
"latency_ms": 7883.9,
"answer": {
"type": "choice",
"choice": "suporte_tecnico",
"confidence": 0.9550325369120006,
"probabilities": {
"pagamento": 0.0009793333603067293,
"cancelamento": 0.002014121845135763,
"suporte_tecnico": 0.9662744026840004,
"sem_correspondencia": 0.030732142110557085
}
},
"pred_permuted": "suporte_tecnico",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:sem_correspondencia",
"gold": "sem_correspondencia",
"pred": "sem_correspondencia",
"latency_ms": 7948.3,
"answer": {
"type": "choice",
"choice": "sem_correspondencia",
"confidence": 0.9885549303273677,
"probabilities": {
"pagamento": 0.0036670988032871665,
"cancelamento": 0.0022098453219148453,
"suporte_tecnico": 0.002706858129272241,
"sem_correspondencia": 0.9914161977455258
}
},
"pred_permuted": "sem_correspondencia",
"permutation_invariant": true,
"correct": true
},
{
"name": "noul:clara",
"gold": "true",
"pred": "true",
"latency_ms": 6893.7,
"answer": {
"type": "noul",
"noul": 0.9479776743420092
},
"correct": true
},
{
"name": "noul:vaga",
"gold": "false",
"pred": "true",
"latency_ms": 6730.7,
"answer": {
"type": "noul",
"noul": 0.5076455176696537
},
"correct": false
},
{
"name": "score:alta",
"gold": "2",
"pred": "2",
"latency_ms": 6445.7,
"answer": {
"type": "score",
"score": 1.9453121647986211,
"confidence": 0.9510302861376827,
"probabilities": {
"0": 0.02204135929316713,
"1": 0.01060511661504444,
"2": 0.9673535240917883
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": true
},
{
"name": "score:baixa",
"gold": "0",
"pred": "0",
"latency_ms": 6312.4,
"answer": {
"type": "score",
"score": 0.2695747926209778,
"confidence": 0.6046525975421342,
"probabilities": {
"0": 0.7364350650280895,
"1": 0.25755507732284305,
"2": 0.006009857649067364
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": true
}
],
"summary": {
"contract_violations": [],
"sanity_correct": "7/8",
"permutation_invariant": "4/4"
}
}

219
reports/sanity/semif-cpu.json Normal file
View File

@@ -0,0 +1,219 @@
{
"candidate": {
"name": "semif",
"model_id": "semif@851bf6e806ef",
"revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
"adapter_version": "0.1.0",
"device": "cpu",
"context_limit": 8192,
"probabilities_source": "native"
},
"device": "cpu",
"host": "host-local",
"load_seconds": 6.55,
"conformance": [
{
"name": "valid_choice_noul",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_score",
"expected": "ok",
"got": "ok"
},
{
"name": "valid_state_object",
"expected": "ok",
"got": "ok"
},
{
"name": "invalid_17_options",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_single_option",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_noul_missing_false",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_unknown_type",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_empty_state",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_no_questions",
"expected": "invalid_request",
"got": "invalid_request"
},
{
"name": "invalid_extra_field",
"expected": "invalid_request",
"got": "invalid_request"
}
],
"sanity": [
{
"name": "choice:pagamento",
"gold": "pagamento",
"pred": "pagamento",
"latency_ms": 15726.0,
"answer": {
"type": "choice",
"choice": "pagamento",
"confidence": 0.9999576137177726,
"probabilities": {
"pagamento": 0.9999576137177726,
"cancelamento": 2.753528218203348e-05,
"suporte_tecnico": 6.962009617274983e-06,
"sem_correspondencia": 7.888990428051513e-06
}
},
"pred_permuted": "pagamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:cancelamento",
"gold": "cancelamento",
"pred": "cancelamento",
"latency_ms": 16084.8,
"answer": {
"type": "choice",
"choice": "cancelamento",
"confidence": 0.9982468109808258,
"probabilities": {
"pagamento": 0.001700633028283725,
"cancelamento": 0.9982468109808258,
"suporte_tecnico": 3.114818042829917e-05,
"sem_correspondencia": 2.1407810462216812e-05
}
},
"pred_permuted": "cancelamento",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:suporte_tecnico",
"gold": "suporte_tecnico",
"pred": "suporte_tecnico",
"latency_ms": 15810.0,
"answer": {
"type": "choice",
"choice": "suporte_tecnico",
"confidence": 0.9985997601915109,
"probabilities": {
"pagamento": 0.0013249226202503434,
"cancelamento": 3.5307991870190554e-05,
"suporte_tecnico": 0.9985997601915109,
"sem_correspondencia": 4.0009196368602506e-05
}
},
"pred_permuted": "suporte_tecnico",
"permutation_invariant": true,
"correct": true
},
{
"name": "choice:sem_correspondencia",
"gold": "sem_correspondencia",
"pred": "sem_correspondencia",
"latency_ms": 15911.3,
"answer": {
"type": "choice",
"choice": "sem_correspondencia",
"confidence": 0.9868525351071245,
"probabilities": {
"pagamento": 0.0096747618254165,
"cancelamento": 0.0027718656714616763,
"suporte_tecnico": 0.0007008373959974223,
"sem_correspondencia": 0.9868525351071245
}
},
"pred_permuted": "sem_correspondencia",
"permutation_invariant": true,
"correct": true
},
{
"name": "noul:clara",
"gold": "true",
"pred": "true",
"latency_ms": 13647.4,
"answer": {
"type": "noul",
"noul": 0.9465966702001758
},
"correct": true
},
{
"name": "noul:vaga",
"gold": "false",
"pred": "false",
"latency_ms": 13233.5,
"answer": {
"type": "noul",
"noul": 0.014063627043245475
},
"correct": true
},
{
"name": "score:alta",
"gold": "2",
"pred": "2",
"latency_ms": 13650.8,
"answer": {
"type": "score",
"score": 1.992728895910905,
"confidence": 0.9963221948103022,
"probabilities": {
"0": 0.003593298899397119,
"1": 8.450629030069868e-05,
"2": 0.9963221948103022
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": true
},
{
"name": "score:baixa",
"gold": "0",
"pred": "0",
"latency_ms": 13731.5,
"answer": {
"type": "score",
"score": 0.6879537693865432,
"confidence": 0.4373487435378189,
"probabilities": {
"0": 0.4373487435378189,
"1": 0.4373487435378189,
"2": 0.12530251292436217
},
"legend": {
"0": "Baixa",
"1": "Média",
"2": "Alta"
}
},
"correct": true
}
],
"summary": {
"contract_violations": [],
"sanity_correct": "8/8",
"permutation_invariant": "4/4"
}
}