self-directed-learning/scripts/eval.py

#!/usr/bin/env python3
"""
Adaptive Recall eval harness — Phase 1F.

Runs the same queries against UMP-only baseline (POST :4317/ump/recall) and
the sidecar fused recall (POST :4380/recall), computes recall@1/3/5, MRR,
and latency p50/p95/p99. Acceptance for Phase 1: sidecar recall@3 must
exceed baseline by >30%.

Usage:
  python3 eval.py                    # full eval, prints results table
  python3 eval.py --json             # machine-readable JSON
  python3 eval.py --weights '{"ump":1.0,"vector":2.0}'  # tune RRF channel weights

Each query in eval/queries.py needs a real `expected_id` (a UMP URN). If
the expected urn doesn't exist or returns null, we treat that query as a
known-gap and skip it from the metric (with a warning in the report).

Output columns:
  query    expected_id  baseline_rank  sidecar_rank  baseline_ms  sidecar_ms
  PLUS summary table at the bottom with recall@1/3/5, MRR, latency percentiles.
"""

import argparse
import json
import os
import statistics
import sys
import time
import urllib.request

# Local imports — eval/queries.py is one directory up
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "eval"))
from queries import QUERIES

UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317")
SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380")


def http_post_json(url, body, timeout=30):
    """POST JSON. Returns (status, parsed_body_or_None, elapsed_ms)."""
    data = json.dumps(body).encode()
    req = urllib.request.Request(
        url, data=data,
        headers={"content-type": "application/json"},
        method="POST",
    )
    t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
            try:
                return resp.status, json.loads(raw), (time.time() - t0) * 1000
            except json.JSONDecodeError:
                return resp.status, None, (time.time() - t0) * 1000
    except Exception as e:
        return 0, {"error": repr(e)}, (time.time() - t0) * 1000


def baseline_recall(query, limit=10):
    """Run query against UMP only. Returns (rank_of_expected_urn_or_None, elapsed_ms)."""
    status, body, ms = http_post_json(
        f"{UMP_URL}/ump/recall",
        {"query": query, "limit": limit},
    )
    if status != 200 or not body:
        return None, ms, f"baseline error: status={status} body={body}"
    hits = body.get("results", [])
    for idx, r in enumerate(hits):
        if r.get("record", {}).get("id") == expected_id_for_query(query):
            return idx + 1, ms, "ok"
    return None, ms, "not in top-K"


def sidecar_recall(query, limit=10, weights=None):
    """Run query against sidecar (RRF fused). Returns (rank, ms, status)."""
    payload = {"query": query, "limit": limit}
    if weights:
        payload["weights"] = weights
    status, body, ms = http_post_json(
        f"{SIDECAR_URL}/recall",
        payload,
        timeout=60,
    )
    if status != 200 or not body:
        return None, ms, f"sidecar error: status={status} body={body}"
    hits = body.get("hits", [])
    exp = expected_id_for_query(query)
    for idx, h in enumerate(hits):
        if h.get("urn") == exp:
            return idx + 1, ms, "ok"
    return None, ms, "not in top-K"


def expected_id_for_query(query):
    """Look up the expected urn from the QUERIES table."""
    for q in QUERIES:
        if q["query"] == query:
            return q.get("expected_id")
    return None


def recall_at_k(ranks, k):
    """Given a list of ranks (None if not in top-K), what fraction made top-K?"""
    hits = sum(1 for r in ranks if r is not None and r <= k)
    return hits / len(ranks) if ranks else 0


def mrr(ranks):
    """Mean reciprocal rank over ranks."""
    if not ranks:
        return 0
    total = sum(1.0 / r for r in ranks if r is not None)
    return total / len(ranks)


def percentile(values, p):
    """Nearest-rank percentile, simple and dependency-free."""
    if not values:
        return 0
    s = sorted(values)
    idx = max(0, min(len(s) - 1, int(len(s) * p / 100)))
    return s[idx]


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--json", action="store_true", help="machine-readable output")
    parser.add_argument("--limit", type=int, default=10, help="top-K for retrieval (default 10)")
    parser.add_argument("--weights", type=str, default=None, help="RRF channel weights as JSON")
    args = parser.parse_args()

    weights = json.loads(args.weights) if args.weights else None

    results = []
    baseline_ranks = []
    sidecar_ranks = []
    baseline_latencies = []
    sidecar_latencies = []
    skipped = []

    for q in QUERIES:
        query = q["query"]
        exp = q.get("expected_id")
        if not exp:
            skipped.append({"query": query, "reason": "no expected_id"})
            continue

        b_rank, b_ms, b_status = baseline_recall(query, limit=args.limit)
        s_rank, s_ms, s_status = sidecar_recall(query, limit=args.limit, weights=weights)

        baseline_ranks.append(b_rank)
        sidecar_ranks.append(s_rank)
        baseline_latencies.append(b_ms)
        sidecar_latencies.append(s_ms)

        results.append({
            "query": query,
            "expected_id": exp,
            "baseline_rank": b_rank,
            "sidecar_rank": s_rank,
            "baseline_ms": b_ms,
            "sidecar_ms": s_ms,
            "baseline_status": b_status,
            "sidecar_status": s_status,
        })

    # Metrics
    metrics = {
        "baseline": {
            "recall@1": recall_at_k(baseline_ranks, 1),
            "recall@3": recall_at_k(baseline_ranks, 3),
            "recall@5": recall_at_k(baseline_ranks, 5),
            "mrr": mrr(baseline_ranks),
            "latency_p50_ms": percentile(baseline_latencies, 50),
            "latency_p95_ms": percentile(baseline_latencies, 95),
            "latency_p99_ms": percentile(baseline_latencies, 99),
        },
        "sidecar": {
            "recall@1": recall_at_k(sidecar_ranks, 1),
            "recall@3": recall_at_k(sidecar_ranks, 3),
            "recall@5": recall_at_k(sidecar_ranks, 5),
            "mrr": mrr(sidecar_ranks),
            "latency_p50_ms": percentile(sidecar_latencies, 50),
            "latency_p95_ms": percentile(sidecar_latencies, 95),
            "latency_p99_ms": percentile(sidecar_latencies, 99),
        },
    }
    # Acceptance gate
    baseline_r3 = metrics["baseline"]["recall@3"]
    sidecar_r3 = metrics["sidecar"]["recall@3"]
    if baseline_r3 > 0:
        improvement_pct = ((sidecar_r3 - baseline_r3) / baseline_r3) * 100
    else:
        improvement_pct = float("inf") if sidecar_r3 > 0 else 0
    metrics["acceptance_recall@3_improvement_pct"] = improvement_pct
    metrics["acceptance_met"] = improvement_pct > 30

    output = {
        "queries_evaluated": len(results),
        "queries_skipped": len(skipped),
        "metrics": metrics,
        "results": results,
        "skipped": skipped,
        "weights": weights or {"ump": 1.0, "vector": 1.0},
    }

    if args.json:
        print(json.dumps(output, indent=2))
    else:
        print(f"\nAdaptive Recall Eval — Phase 1F")
        print(f"{'='*60}")
        print(f"Queries evaluated: {len(results)} | Skipped: {len(skipped)}")
        print(f"\n{'metric':<28} {'baseline':>10} {'sidecar':>10} {'delta':>10}")
        print(f"{'-'*60}")
        for k in ["recall@1", "recall@3", "recall@5", "mrr"]:
            b = metrics["baseline"][k]
            s = metrics["sidecar"][k]
            delta = s - b
            print(f"{k:<28} {b:>10.2%} {s:>10.2%} {delta:>+10.2%}")
        for k in ["latency_p50_ms", "latency_p95_ms", "latency_p99_ms"]:
            b = metrics["baseline"][k]
            s = metrics["sidecar"][k]
            delta = s - b
            print(f"{k:<28} {b:>10.0f} {s:>10.0f} {delta:>+10.0f}")
        print(f"\nAcceptance: recall@3 improvement = {improvement_pct:+.1f}% (target >30%)")
        print(f"Result: {'PASS' if metrics['acceptance_met'] else 'NEEDS WORK'}")
        print(f"\n{'='*60}")
        print(f"Per-query results:")
        print(f"{'query':<55} {'B-rank':>7} {'S-rank':>7} {'B-ms':>7} {'S-ms':>7}")
        print(f"{'-'*83}")
        for r in results:
            b_str = f"{r['baseline_rank']}" if r['baseline_rank'] else "miss"
            s_str = f"{r['sidecar_rank']}" if r['sidecar_rank'] else "miss"
            q_short = r['query'][:54]
            print(f"{q_short:<55} {b_str:>7} {s_str:>7} {r['baseline_ms']:>7.0f} {r['sidecar_ms']:>7.0f}")
        if skipped:
            print(f"\nSkipped: {skipped}")


if __name__ == "__main__":
    main()