mcloh/decision-models-benchmark
1
0
Fork 0
mirror of https://github.com/mcloh/decision-models-benchmark.git synced 2026-11-06 06:14:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

setup environment

Browse Source
This commit is contained in:
Marcos Lohmann
2026-10-07 11:14:29 -03:00
commit aeee6c7de1
25 changed files with 1237 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
contracts/jev/schemas/request.schema.json Normal file
View File

@@ -0,0 +1,79 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "jev-compat-v1/request.schema.json",
"title": "JEV systemone request (jev-compat-v1)",
"type": "object",
"required": ["model", "state", "questions"],
"additionalProperties": false,
"properties": {
"model": {"type": "string", "minLength": 1},
"state": {
"oneOf": [
{"type": "string", "minLength": 1},
{"type": "object", "minProperties": 1},
{"type": "array", "minItems": 1}
]
},
"questions": {
"type": "object",
"minProperties": 1,
"additionalProperties": {"$ref": "#/$defs/question"}
}
},
"$defs": {
"question": {
"type": "object",
"required": ["type", "instructions", "criteria"],
"additionalProperties": false,
"properties": {
"type": {"enum": ["choice", "noul", "score"]},
"instructions": {"type": "string", "minLength": 1},
"criteria": {}
},
"allOf": [
{
"if": {"properties": {"type": {"const": "choice"}}},
"then": {
"properties": {
"criteria": {
"type": "object",
"minProperties": 2,
"maxProperties": 20,
"additionalProperties": {"type": "string", "minLength": 1}
}
}
}
},
{
"if": {"properties": {"type": {"const": "noul"}}},
"then": {
"properties": {
"criteria": {
"type": "object",
"required": ["true", "false"],
"additionalProperties": false,
"properties": {
"true": {"type": "string", "minLength": 1},
"false": {"type": "string", "minLength": 1}
}
}
}
}
},
{
"if": {"properties": {"type": {"const": "score"}}},
"then": {
"properties": {
"criteria": {
"type": "array",
"minItems": 2,
"maxItems": 20,
"items": {"type": "string", "minLength": 1}
}
}
}
}
]
}
}
}

74
contracts/jev/schemas/response.schema.json Normal file
View File

@@ -0,0 +1,74 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "jev-compat-v1/response.schema.json",
"title": "JEV systemone response (jev-compat-v1)",
"type": "object",
"required": ["model", "answers", "usage"],
"properties": {
"model": {"type": "string", "minLength": 1},
"answers": {
"type": "object",
"minProperties": 1,
"additionalProperties": {
"oneOf": [
{"$ref": "#/$defs/choice"},
{"$ref": "#/$defs/noul"},
{"$ref": "#/$defs/score"}
]
}
},
"usage": {
"type": "object",
"required": ["input_tokens", "output_tokens"],
"properties": {
"input_tokens": {"type": "integer", "minimum": 0},
"output_tokens": {"const": 0}
}
},
"metadata": {
"type": "object",
"description": "Extensão do benchmark (tarefa 48); ignorada por clientes JEV.",
"properties": {
"latency_ms": {"type": "number", "minimum": 0},
"model_id": {"type": "string"},
"run_id": {"type": "string"},
"adapter_version": {"type": "string"},
"probabilities_source": {"enum": ["native", "normalized_scores", "logprobs", "none"]},
"truncated": {"type": "boolean"}
}
}
},
"$defs": {
"prob": {"type": "number", "minimum": 0, "maximum": 1},
"probs": {"type": "object", "minProperties": 1, "additionalProperties": {"$ref": "#/$defs/prob"}},
"choice": {
"type": "object",
"required": ["type", "choice", "probabilities"],
"properties": {
"type": {"const": "choice"},
"choice": {"type": "string"},
"confidence": {"$ref": "#/$defs/prob"},
"probabilities": {"$ref": "#/$defs/probs"}
}
},
"noul": {
"type": "object",
"required": ["type", "noul"],
"properties": {
"type": {"const": "noul"},
"noul": {"$ref": "#/$defs/prob"}
}
},
"score": {
"type": "object",
"required": ["type", "score", "probabilities", "legend"],
"properties": {
"type": {"const": "score"},
"score": {"type": "number", "minimum": 0},
"confidence": {"$ref": "#/$defs/prob"},
"probabilities": {"$ref": "#/$defs/probs"},
"legend": {"type": "object", "additionalProperties": {"type": "string"}}
}
}
}
}