first commit

evaluator/analytics/vloop.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any, Iterable
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def extract_intent(record_or_raw: Any) -> str | None:
+    """Extract the trace intent used by the TIM specification for the deterministic loop metric."""
+    if isinstance(record_or_raw, dict):
+        metadata = _as_dict(record_or_raw.get("metadata"))
+        nested_raw = _as_dict(record_or_raw.get("raw"))
+    else:
+        metadata = _as_dict(getattr(record_or_raw, "metadata", None))
+        nested_raw = _as_dict(getattr(record_or_raw, "raw", None))
+
+    trace = _as_dict(nested_raw.get("trace"))
+    detail = _as_dict(nested_raw.get("detail"))
+    trace_meta = _as_dict(trace.get("metadata"))
+    detail_meta = _as_dict(detail.get("metadata"))
+
+    value = (
+        metadata.get("intent")
+        or trace_meta.get("intent")
+        or detail_meta.get("intent")
+    )
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def max_consecutive_intent_streak(records: Iterable[Any]) -> int:
+    """Maximum consecutive streak of the same metadata.intent in trace order."""
+    maximum = 0
+    current = 0
+    previous: str | None = None
+
+    for record in records:
+        intent = extract_intent(record)
+        if not intent:
+            previous = None
+            current = 0
+            continue
+        if intent == previous:
+            current += 1
+        else:
+            previous = intent
+            current = 1
+        maximum = max(maximum, current)
+
+    return maximum
+
+
+def session_loop_values(records: Iterable[Any], session_id_getter=None) -> dict[str, int]:
+    grouped: dict[str, list[Any]] = defaultdict(list)
+    for record in records:
+        if session_id_getter:
+            sid = session_id_getter(record)
+        elif isinstance(record, dict):
+            sid = record.get("session_id") or record.get("sessionId")
+        else:
+            sid = getattr(record, "session_id", None)
+        if sid:
+            grouped[str(sid)].append(record)
+    return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()}
+
+
+# Backward-compatible name used by older integrations. For a single trace the
+# streak is either 0 (no intent) or 1 (intent present); the official metric is
+# session-level and should use max_consecutive_intent_streak over all traces.
+def vloop_flag(record_or_raw: Any) -> int:
+    return max_consecutive_intent_streak([record_or_raw])