#!/usr/bin/env python3 """ Adaptive Recall eval harness — Phase 1F. Runs the same queries against UMP-only baseline (POST :4317/ump/recall) and the sidecar fused recall (POST :4380/recall), computes recall@1/3/5, MRR, and latency p50/p95/p99. Acceptance for Phase 1: sidecar recall@3 must exceed baseline by >30%. Usage: python3 eval.py # full eval, prints results table python3 eval.py --json # machine-readable JSON python3 eval.py --weights '{"ump":1.0,"vector":2.0}' # tune RRF channel weights Each query in eval/queries.py needs a real `expected_id` (a UMP URN). If the expected urn doesn't exist or returns null, we treat that query as a known-gap and skip it from the metric (with a warning in the report). Output columns: query expected_id baseline_rank sidecar_rank baseline_ms sidecar_ms PLUS summary table at the bottom with recall@1/3/5, MRR, latency percentiles. """ import argparse import json import os import statistics import sys import time import urllib.request # Local imports — eval/queries.py is one directory up sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "eval")) from queries import QUERIES UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317") SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380") def http_post_json(url, body, timeout=30): """POST JSON. Returns (status, parsed_body_or_None, elapsed_ms).""" data = json.dumps(body).encode() req = urllib.request.Request( url, data=data, headers={"content-type": "application/json"}, method="POST", ) t0 = time.time() try: with urllib.request.urlopen(req, timeout=timeout) as resp: raw = resp.read() try: return resp.status, json.loads(raw), (time.time() - t0) * 1000 except json.JSONDecodeError: return resp.status, None, (time.time() - t0) * 1000 except Exception as e: return 0, {"error": repr(e)}, (time.time() - t0) * 1000 def baseline_recall(query, limit=10): """Run query against UMP only. Returns (rank_of_expected_urn_or_None, elapsed_ms).""" status, body, ms = http_post_json( f"{UMP_URL}/ump/recall", {"query": query, "limit": limit}, ) if status != 200 or not body: return None, ms, f"baseline error: status={status} body={body}" hits = body.get("results", []) for idx, r in enumerate(hits): if r.get("record", {}).get("id") == expected_id_for_query(query): return idx + 1, ms, "ok" return None, ms, "not in top-K" def sidecar_recall(query, limit=10, weights=None): """Run query against sidecar (RRF fused). Returns (rank, ms, status).""" payload = {"query": query, "limit": limit} if weights: payload["weights"] = weights status, body, ms = http_post_json( f"{SIDECAR_URL}/recall", payload, timeout=60, ) if status != 200 or not body: return None, ms, f"sidecar error: status={status} body={body}" hits = body.get("hits", []) exp = expected_id_for_query(query) for idx, h in enumerate(hits): if h.get("urn") == exp: return idx + 1, ms, "ok" return None, ms, "not in top-K" def expected_id_for_query(query): """Look up the expected urn from the QUERIES table.""" for q in QUERIES: if q["query"] == query: return q.get("expected_id") return None def recall_at_k(ranks, k): """Given a list of ranks (None if not in top-K), what fraction made top-K?""" hits = sum(1 for r in ranks if r is not None and r <= k) return hits / len(ranks) if ranks else 0 def mrr(ranks): """Mean reciprocal rank over ranks.""" if not ranks: return 0 total = sum(1.0 / r for r in ranks if r is not None) return total / len(ranks) def percentile(values, p): """Nearest-rank percentile, simple and dependency-free.""" if not values: return 0 s = sorted(values) idx = max(0, min(len(s) - 1, int(len(s) * p / 100))) return s[idx] def main(): parser = argparse.ArgumentParser() parser.add_argument("--json", action="store_true", help="machine-readable output") parser.add_argument("--limit", type=int, default=10, help="top-K for retrieval (default 10)") parser.add_argument("--weights", type=str, default=None, help="RRF channel weights as JSON") args = parser.parse_args() weights = json.loads(args.weights) if args.weights else None results = [] baseline_ranks = [] sidecar_ranks = [] baseline_latencies = [] sidecar_latencies = [] skipped = [] for q in QUERIES: query = q["query"] exp = q.get("expected_id") if not exp: skipped.append({"query": query, "reason": "no expected_id"}) continue b_rank, b_ms, b_status = baseline_recall(query, limit=args.limit) s_rank, s_ms, s_status = sidecar_recall(query, limit=args.limit, weights=weights) baseline_ranks.append(b_rank) sidecar_ranks.append(s_rank) baseline_latencies.append(b_ms) sidecar_latencies.append(s_ms) results.append({ "query": query, "expected_id": exp, "baseline_rank": b_rank, "sidecar_rank": s_rank, "baseline_ms": b_ms, "sidecar_ms": s_ms, "baseline_status": b_status, "sidecar_status": s_status, }) # Metrics metrics = { "baseline": { "recall@1": recall_at_k(baseline_ranks, 1), "recall@3": recall_at_k(baseline_ranks, 3), "recall@5": recall_at_k(baseline_ranks, 5), "mrr": mrr(baseline_ranks), "latency_p50_ms": percentile(baseline_latencies, 50), "latency_p95_ms": percentile(baseline_latencies, 95), "latency_p99_ms": percentile(baseline_latencies, 99), }, "sidecar": { "recall@1": recall_at_k(sidecar_ranks, 1), "recall@3": recall_at_k(sidecar_ranks, 3), "recall@5": recall_at_k(sidecar_ranks, 5), "mrr": mrr(sidecar_ranks), "latency_p50_ms": percentile(sidecar_latencies, 50), "latency_p95_ms": percentile(sidecar_latencies, 95), "latency_p99_ms": percentile(sidecar_latencies, 99), }, } # Acceptance gate baseline_r3 = metrics["baseline"]["recall@3"] sidecar_r3 = metrics["sidecar"]["recall@3"] if baseline_r3 > 0: improvement_pct = ((sidecar_r3 - baseline_r3) / baseline_r3) * 100 else: improvement_pct = float("inf") if sidecar_r3 > 0 else 0 metrics["acceptance_recall@3_improvement_pct"] = improvement_pct metrics["acceptance_met"] = improvement_pct > 30 output = { "queries_evaluated": len(results), "queries_skipped": len(skipped), "metrics": metrics, "results": results, "skipped": skipped, "weights": weights or {"ump": 1.0, "vector": 1.0}, } if args.json: print(json.dumps(output, indent=2)) else: print(f"\nAdaptive Recall Eval — Phase 1F") print(f"{'='*60}") print(f"Queries evaluated: {len(results)} | Skipped: {len(skipped)}") print(f"\n{'metric':<28} {'baseline':>10} {'sidecar':>10} {'delta':>10}") print(f"{'-'*60}") for k in ["recall@1", "recall@3", "recall@5", "mrr"]: b = metrics["baseline"][k] s = metrics["sidecar"][k] delta = s - b print(f"{k:<28} {b:>10.2%} {s:>10.2%} {delta:>+10.2%}") for k in ["latency_p50_ms", "latency_p95_ms", "latency_p99_ms"]: b = metrics["baseline"][k] s = metrics["sidecar"][k] delta = s - b print(f"{k:<28} {b:>10.0f} {s:>10.0f} {delta:>+10.0f}") print(f"\nAcceptance: recall@3 improvement = {improvement_pct:+.1f}% (target >30%)") print(f"Result: {'PASS' if metrics['acceptance_met'] else 'NEEDS WORK'}") print(f"\n{'='*60}") print(f"Per-query results:") print(f"{'query':<55} {'B-rank':>7} {'S-rank':>7} {'B-ms':>7} {'S-ms':>7}") print(f"{'-'*83}") for r in results: b_str = f"{r['baseline_rank']}" if r['baseline_rank'] else "miss" s_str = f"{r['sidecar_rank']}" if r['sidecar_rank'] else "miss" q_short = r['query'][:54] print(f"{q_short:<55} {b_str:>7} {s_str:>7} {r['baseline_ms']:>7.0f} {r['sidecar_ms']:>7.0f}") if skipped: print(f"\nSkipped: {skipped}") if __name__ == "__main__": main()