mirror of
https://github.com/hoshikawa2/agent_platform_oci.git
synced 2026-07-09 22:04:21 +00:00
81 lines
3.9 KiB
Python
81 lines
3.9 KiB
Python
from __future__ import annotations
|
|
import asyncio
|
|
from datetime import datetime, timedelta
|
|
import typer
|
|
from rich import print
|
|
from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn
|
|
from evaluator.config.agents import load_agents
|
|
from evaluator.engine import EvaluationEngine
|
|
from evaluator.persistence.repository import EvaluationRepository
|
|
from evaluator.config.settings import settings
|
|
|
|
app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator')
|
|
|
|
|
|
def _run_progress(coro_factory):
|
|
async def runner():
|
|
state={'run_id': None}
|
|
with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress:
|
|
task=progress.add_task('evaluation', total=1, stage='starting')
|
|
async def cb(event):
|
|
state['run_id'] = event.get('run_id') or state['run_id']
|
|
stage = event.get('stage','')
|
|
msg = event.get('message','')
|
|
if state['run_id']:
|
|
snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1)
|
|
total=int(snap.get('total_items') or 0) or 1
|
|
done=int(snap.get('done_items') or 0)
|
|
progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120])
|
|
result = await coro_factory(cb)
|
|
progress.update(task, completed=1, total=1, stage='finished')
|
|
return result
|
|
return asyncio.run(runner())
|
|
|
|
@app.command("reset-db")
|
|
def reset_db():
|
|
repo = EvaluationRepository(auto_init_schema=False)
|
|
repo.store.drop_schema()
|
|
repo.store._init_schema()
|
|
print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."})
|
|
|
|
@app.command('init-db')
|
|
def init_db():
|
|
EvaluationRepository(auto_init_schema=True)
|
|
print({'status':'OK','message':'schema checked/created'})
|
|
|
|
@app.command('show-config')
|
|
def show_config():
|
|
print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path})
|
|
|
|
@app.command('run')
|
|
def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True):
|
|
if show_progress:
|
|
result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit))
|
|
else:
|
|
result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit))
|
|
print(result)
|
|
|
|
@app.command('run-agents')
|
|
def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None):
|
|
async def main():
|
|
results=[]
|
|
now=datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
|
|
for agent in load_agents():
|
|
if agent_id and agent.agent_id != agent_id: continue
|
|
start = now - timedelta(days=agent.days_back)
|
|
engine=EvaluationEngine()
|
|
results.append(await engine.run_agent(agent, start, now, source=source, limit=limit))
|
|
return results
|
|
print(asyncio.run(main()))
|
|
|
|
@app.command('progress')
|
|
def progress(run_id: str, events: int=20):
|
|
print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events)))
|
|
|
|
@app.command('runs')
|
|
def runs(limit: int=20):
|
|
print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit)))
|
|
|
|
if __name__ == '__main__':
|
|
app()
|