I turn LLM prototypes that work in notebooks into features that work in production — evals, observability, and all.

1from ragas import evaluate
2from ragas.metrics import faithfulness, answer_relevancy
3 
4# Custom domain metrics
5class CitationAccuracy:
6    def __call__(self, row):
7        citations = extract_citations(row.answer)
8        verified = verify_against_source(citations, row.contexts)
9        return len(verified) / len(citations) if citations else 1.0
10 
11eval_dataset = load_dataset("legal-qa")
12results = evaluate(
13    eval_dataset,
14    metrics=[faithfulness, answer_relevancy, CitationAccuracy()],
15)
16 
17# CI gate: fail if hallucinations > 5%
18assert results['CitationAccuracy'] > 0.95

1from temporalio import workflow
2from langgraph.graph import StateGraph
3 
4class ResearchWorkflow:
5    @workflow.run
6    async def run(self, query: str) -> Report:
7        # Structured agent with checkpoints
8        planner = create_planner_agent()
9        researcher = create_researcher_agent()
10        writer = create_writer_agent()
11 
12        plan = await planner.aplan(query)
13        findings = await researcher.aresearch(plan)
14        report = await writer.awrite(findings)
15 
16        # Human-in-the-loop for compliance
17        if report.risk_level == "high":
18            report = await workflow.wait_for_human_review(report)
19 
20        return report

1@app.post("/eval/run")
2async def run_evaluation(request: EvalRequest):
3    dataset = load_dataset(request.dataset_id)
4    results = []
5 
6    for sample in dataset:
7        output = await run_pipeline(sample.input, request.prompt_version)
8        metrics = compute_metrics(sample.expected, output)
9        results.append({**sample.dict(), **metrics, "output": output})
10 
11    # Regression detection
12    baseline = get_baseline(request.prompt_version)
13    regressions = detect_regressions(results, baseline)
14 
15    await store_results(request.run_id, results, regressions)
16    return {"run_id": request.run_id, "regressions": regressions}