from chronicle import Chronicle
with Chronicle() as client:
example = next((e for e in client.worlds.examples.list().items
if any(t.key == "triage-duplicate-invoice"
for t in e.blueprint.scenario.tasks)), None)
if example is None:
raise RuntimeError("The duplicate-invoice example is not available")
preview = client.worlds.compile(blueprint=example.blueprint)
if preview.problems:
raise RuntimeError(preview.problems)
world = client.worlds.create(blueprint=example.blueprint, idempotency_key="demo-1")
world = client.worlds.wait(world.world.id, timeout=180)
if world.status != "ready":
raise RuntimeError(f"World {world.world.id}: {world.status}; {world.launch_error}")
if world.evaluation is None or world.evaluation.status != "ready":
raise RuntimeError(f"Task setup is not ready for world {world.world.id}; see the Worlds guide")
run = client.evaluations.create(
task_suite_id=world.evaluation.task_suite_id,
agents=["billing-agent@1.0.0"],
idempotency_key="demo-run-1",
)
run = client.evaluations.wait(run.id, timeout=600)
if run.status != "succeeded":
raise RuntimeError(f"Evaluation {run.id}: {run.status}; inspect its trials")
for result in client.evaluations.results(run.id).task_results:
print(result.task_id, "passed" if result.passed else "failed", result.failed_scorers)