self-directed-learning/scripts/eval_3ch_vs_2ch.py

#!/usr/bin/env python3
"""
Phase 2 eval — compare 3-channel (ump+vector+graph) vs 2-channel (ump+vector).

Method:
1. Run each eval query with GRAPH_ENABLED=true → 3-channel sidecar
2. Run same query with GRAPH_ENABLED=false → 2-channel sidecar
   (achieved by setting weights={graph:0} which zero-weights graph contributions;
    OR by adding a query flag. Simpler: use ?channels=ump,vector if we add that.)

For ground truth: use the verified-queries corpus (queries_verified.py).
For broader coverage: also run unverified queries (queries.py) and report
graph's marginal contribution.

Reports:
  - 3ch hit_rate@K (K=3,5,10)
  - 2ch hit_rate@K
  - delta_lift = 3ch - 2ch per K
  - graph-only wins: queries where 3ch finds GT at rank R, 2ch doesn't
  - 2ch-only wins: queries where 2ch finds GT at rank R, 3ch doesn't
  - Average rank shift for queries both find

Acceptance: 3ch hit_rate@10 >= 2ch hit_rate@10 (graph should not regress).
Stretch:  3ch hit_rate@10 > 2ch hit_rate@10 by >=5pp (graph adds value).
"""
import json
import os
import statistics
import sys
import time
import urllib.parse
import urllib.request

UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317")
SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380")


def http_json(url, body=None, method="GET", timeout=60):
    data = json.dumps(body).encode() if body else None
    req = urllib.request.Request(
        url, data=data,
        headers={"content-type": "application/json"},
        method=method,
    )
    t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
            return resp.status, json.loads(raw) if raw else None, (time.time() - t0) * 1000
    except Exception as e:
        return 0, {"error": repr(e)}, (time.time() - t0) * 1000


def recall_sidecar(query, limit=10, with_graph=True):
    """Run sidecar /recall, optionally zero-weighting the graph channel."""
    weights = None if with_graph else {"ump": 1.0, "vector": 1.0, "graph": 0.0}
    body = {"query": query, "limit": limit}
    if weights:
        body["weights"] = weights
    return http_json(f"{SIDECAR_URL}/recall", body, method="POST")


def recall_baseline(query, limit=10):
    """Raw UMP recall (FTS5 + recency + salience)."""
    return http_json(f"{UMP_URL}/ump/recall", {"query": query, "limit": limit}, method="POST")


def rank_of(hits, urn):
    for idx, h in enumerate(hits):
        if h.get("urn") == urn or h.get("record", {}).get("id") == urn:
            return idx + 1
    return None


def main():
    # Try verified queries first, fall back to full queries.
    qpath = None
    for candidate in [
        "/root/ump-recall/eval/queries.py",
        "/root/ump-recall/eval/queries_verified.py",
    ]:
        if os.path.exists(candidate):
            qpath = candidate
            break
    if not qpath:
        print("No queries file found.")
        sys.exit(1)

    sys.path.insert(0, os.path.dirname(qpath))
    queries_module = os.path.basename(qpath).replace(".py", "")
    # Tolerate either QUERIES or QUERIES_VERIFIED constant name.
    mod = __import__(queries_module)
    queries = getattr(mod, "QUERIES", None) or getattr(mod, "QUERIES_VERIFIED", None)
    if not queries:
        print(f"No QUERIES / QUERIES_VERIFIED found in {qpath}")
        sys.exit(1)
    print(f"Loaded {len(queries)} queries from {qpath}\n")

    results = []
    for q in queries:
        query = q["query"]
        gt_urn = q.get("expected_id")
        if not gt_urn:
            continue

        # 3-channel
        s3, b3, t3 = recall_sidecar(query, 10, with_graph=True)
        r3 = rank_of(b3.get("hits", []), gt_urn) if s3 == 200 else None

        # 2-channel
        s2, b2, t2 = recall_sidecar(query, 10, with_graph=False)
        r2 = rank_of(b2.get("hits", []), gt_urn) if s2 == 200 else None

        # baseline
        sB, bB, tB = recall_baseline(query, 10)
        rB = None
        if sB == 200:
            for idx, r in enumerate(bB.get("results", [])):
                if r.get("record", {}).get("id") == gt_urn:
                    rB = idx + 1
                    break

        results.append({
            "query": query[:60],
            "gt_urn": gt_urn[-25:],
            "baseline_rank": rB,
            "two_ch_rank": r2,
            "three_ch_rank": r3,
            "t_baseline_ms": round(tB, 1),
            "t_two_ch_ms": round(t2, 1),
            "t_three_ch_ms": round(t3, 1),
        })

    n = len(results)
    if not n:
        print("No queries with expected_id.")
        return

    def hit(rank, k):
        return 1 if (rank is not None and rank <= k) else 0

    for k in [3, 5, 10]:
        b = sum(hit(r["baseline_rank"], k) for r in results) / n
        c2 = sum(hit(r["two_ch_rank"], k) for r in results) / n
        c3 = sum(hit(r["three_ch_rank"], k) for r in results) / n
        print(f"Recall@{k:>2}:  baseline={b:.2%}  2ch={c2:.2%}  3ch={c3:.2%}  Δ(3-2)={c3-c2:+.2%}")

    # Graph's marginal contribution
    g_only = sum(1 for r in results if r["two_ch_rank"] is None and r["three_ch_rank"] is not None)
    both_found = sum(1 for r in results if r["two_ch_rank"] is not None and r["three_ch_rank"] is not None)
    rank_improved = sum(1 for r in results if r["two_ch_rank"] is not None and r["three_ch_rank"] is not None and r["three_ch_rank"] < r["two_ch_rank"])
    rank_regressed = sum(1 for r in results if r["two_ch_rank"] is not None and r["three_ch_rank"] is not None and r["three_ch_rank"] > r["two_ch_rank"])
    two_only = sum(1 for r in results if r["two_ch_rank"] is not None and r["three_ch_rank"] is None)

    print(f"\nGraph channel contribution:")
    print(f"  Graph-only wins (3ch finds, 2ch misses):  {g_only}")
    print(f"  2ch-only wins (3ch misses, 2ch finds):    {two_only}")
    print(f"  Both found:                                {both_found}")
    print(f"    Rank improved by graph:                  {rank_improved}")
    print(f"    Rank regressed by graph:                 {rank_regressed}")

    # Latency
    avg_lat_2 = statistics.mean(r["t_two_ch_ms"] for r in results)
    avg_lat_3 = statistics.mean(r["t_three_ch_ms"] for r in results)
    avg_lat_B = statistics.mean(r["t_baseline_ms"] for r in results)
    print(f"\nAvg latency: baseline={avg_lat_B:.0f}ms  2ch={avg_lat_2:.0f}ms  3ch={avg_lat_3:.0f}ms (Δ={avg_lat_3-avg_lat_2:+.0f}ms)")

    print(f"\nPer-query (top {min(n, 30)}):")
    print(f"{'query':<55} {'BL':>4} {'2ch':>4} {'3ch':>4}  {'2ch-ms':>6} {'3ch-ms':>6}")
    for r in results[:30]:
        b_s = str(r["baseline_rank"]) if r["baseline_rank"] else "-"
        c2_s = str(r["two_ch_rank"]) if r["two_ch_rank"] else "-"
        c3_s = str(r["three_ch_rank"]) if r["three_ch_rank"] else "-"
        print(f"{r['query'][:54]:<55} {b_s:>4} {c2_s:>4} {c3_s:>4}  {r['t_two_ch_ms']:>6.0f} {r['t_three_ch_ms']:>6.0f}")


if __name__ == "__main__":
    main()