self-directed-learning/scripts/eval_selfboot.py

#!/usr/bin/env python3
"""
Phase 1F eval — self-bootstrapping version.

The original eval failed because hand-picked expected_ids were guesses that
didn't match real UMP records. This version:

1. Runs each query against the sidecar's vector channel.
2. Takes the top-1 vector hit as "ground truth" (the query semantically
   matches SOMETHING — that's what we test for).
3. Runs the same query against UMP-only baseline (FTS5 + recency).
4. Measures: did the baseline find the same record? If yes, baseline wins
   on that query. If no, sidecar is the only path that surfaces it.

This measures the REAL question: "does the vector channel find relevant
records that FTS5 misses?" — which is the Adaptive Recall improvement
we're building.

Plus a precision sanity check: of the top-3 sidecar hits, how many have
keywords overlapping with the query? High overlap = vector channel is
finding what we want, not noise.

Output: same format as eval.py.
"""

import argparse
import json
import os
import statistics
import sys
import time
import urllib.parse
import urllib.request

UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317")
SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380")


def http_json(url, body=None, method="GET", timeout=30):
    data = json.dumps(body).encode() if body else None
    req = urllib.request.Request(
        url, data=data,
        headers={"content-type": "application/json"},
        method=method,
    )
    t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
            try:
                return resp.status, json.loads(raw), (time.time() - t0) * 1000
            except json.JSONDecodeError:
                return resp.status, None, (time.time() - t0) * 1000
    except Exception as e:
        return 0, {"error": repr(e)}, (time.time() - t0) * 1000


def run_query(url, payload, timeout=30):
    return http_json(url, payload, method="POST", timeout=timeout)


def fetch_record(urn):
    status, body, _ = http_json(
        f"{UMP_URL}/ump/memory/{urllib.parse.quote(urn, safe='')}",
        method="GET", timeout=5,
    )
    if status != 200:
        return None
    return body.get("record") or body


def recall_at_k(rank, k):
    return 1 if (rank is not None and rank <= k) else 0


def percentile(values, p):
    if not values:
        return 0
    s = sorted(values)
    idx = max(0, min(len(s) - 1, int(len(s) * p / 100)))
    return s[idx]


def overlap_score(query, text):
    """What fraction of query keywords appear in text? (0..1)"""
    if not text:
        return 0
    q_words = {w.lower() for w in query.split() if len(w) > 3}
    if not q_words:
        return 0
    text_lower = text.lower()
    hits = sum(1 for w in q_words if w in text_lower)
    return hits / len(q_words)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--limit", type=int, default=10)
    parser.add_argument("--queries-from", default=None,
                        help="Path to queries.py (default: ../eval/queries.py)")
    args = parser.parse_args()

    if args.queries_from:
        sys.path.insert(0, os.path.dirname(args.queries_from))
        from queries import QUERIES
    else:
        sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "eval"))
        from queries import QUERIES

    # For each query:
    #   1. Sidecar vector channel top-1 = ground truth urn
    #   2. Baseline UMP tries to find it. If found at rank R, baseline recall@R=1.
    #   3. Sidecar RRF tries to find it. Same metric.
    # Plus precision check: top-3 sidecar keyword overlap with query.
    results = []
    baseline_hits = []  # 1/0 per query: did baseline find the ground truth?
    sidecar_hits = []   # 1/0 per query: did sidecar find the ground truth?
    overlap_scores = []  # top-3 sidecar keyword overlap average

    for q in QUERIES:
        query = q["query"]
        # Step 1: get ground truth from vector channel top-1
        status, body, gt_ms = run_query(
            f"{SIDECAR_URL}/recall",
            {"query": query, "limit": args.limit},
        )
        if status != 200 or not body:
            results.append({"query": query, "error": f"sidecar vector failed: {body}"})
            continue
        gt_urn = body["hits"][0]["urn"] if body.get("hits") else None
        gt_subject = (body["hits"][0].get("record") or {}).get("body", {}).get("subject", "")
        gt_text = (body["hits"][0].get("record") or {}).get("body", {}).get("text", "")

        if not gt_urn:
            results.append({"query": query, "error": "vector channel returned no hits"})
            continue

        # Step 2: baseline UMP recall
        status, body, b_ms = run_query(
            f"{UMP_URL}/ump/recall",
            {"query": query, "limit": args.limit},
        )
        baseline_rank = None
        if status == 200 and body:
            for idx, r in enumerate(body.get("results", [])):
                if r.get("record", {}).get("id") == gt_urn:
                    baseline_rank = idx + 1
                    break

        # Step 3: sidecar RRF (already have it, but re-fetch for fair timing)
        status, body, s_ms = run_query(
            f"{SIDECAR_URL}/recall",
            {"query": query, "limit": args.limit},
        )
        sidecar_rank = None
        top3_overlap = 0
        if status == 200 and body:
            for idx, h in enumerate(body.get("hits", [])):
                if h.get("urn") == gt_urn:
                    sidecar_rank = idx + 1
                    break
            # Top-3 keyword overlap
            top3 = body.get("hits", [])[:3]
            overlaps = []
            for h in top3:
                rec = h.get("record") or {}
                text = rec.get("body", {}).get("text", "")[:500]  # first 500 chars
                subject = rec.get("body", {}).get("subject", "")
                overlaps.append(overlap_score(query, f"{subject} {text}"))
            top3_overlap = sum(overlaps) / len(overlaps) if overlaps else 0

        baseline_hits.append(recall_at_k(baseline_rank, 10))
        sidecar_hits.append(recall_at_k(sidecar_rank, 10))
        overlap_scores.append(top3_overlap)

        results.append({
            "query": query,
            "ground_truth_urn": gt_urn[-30:],
            "ground_truth_subject": gt_subject[:50],
            "baseline_rank": baseline_rank,
            "sidecar_rank": sidecar_rank,
            "baseline_ms": b_ms,
            "sidecar_ms": s_ms,
            "sidecar_top3_overlap": round(top3_overlap, 3),
        })

    n = len(results)
    if n == 0:
        print("No results to analyze.")
        return

    # Metrics
    metrics = {
        "n_queries": n,
        "baseline_hit_rate@10": sum(baseline_hits) / n,
        "sidecar_hit_rate@10": sum(sidecar_hits) / n,
        "sidecar_top3_keyword_overlap_avg": sum(overlap_scores) / n,
    }

    # Lift = sidecar hit rate - baseline hit rate
    metrics["hit_rate_lift"] = (
        metrics["sidecar_hit_rate@10"] - metrics["baseline_hit_rate@10"]
    )

    # Output
    print(f"\nAdaptive Recall Eval (self-bootstrapping) — Phase 1F")
    print(f"{'='*70}")
    print(f"Queries evaluated: {n}")
    print(f"\n{'metric':<40} {'value':>15}")
    print(f"{'-'*55}")
    for k, v in metrics.items():
        if isinstance(v, float) and k != "queries_evaluated":
            print(f"{k:<40} {v:>15.3f}")
        else:
            print(f"{k:<40} {v:>15}")

    # Count scenarios
    sidecar_only = sum(1 for r in results if r.get("baseline_rank") is None and r.get("sidecar_rank") is not None)
    baseline_only = sum(1 for r in results if r.get("baseline_rank") is not None and r.get("sidecar_rank") is None)
    both_found = sum(1 for r in results if r.get("baseline_rank") is not None and r.get("sidecar_rank") is not None)
    neither = sum(1 for r in results if r.get("baseline_rank") is None and r.get("sidecar_rank") is None)

    print(f"\nScenario breakdown:")
    print(f"  Both find:        {both_found:>3}")
    print(f"  Sidecar-only:     {sidecar_only:>3}  ← semantic channel found, FTS missed")
    print(f"  Baseline-only:     {baseline_only:>3}")
    print(f"  Neither:          {neither:>3}")

    print(f"\n{'='*70}")
    print(f"Per-query details (GT = vector-channel top-1):")
    print(f"{'query':<55} {'B-rank':>7} {'S-rank':>7} {'top3OL':>7}")
    print(f"{'-'*78}")
    for r in results:
        b_str = f"{r['baseline_rank']}" if r.get("baseline_rank") else "miss"
        s_str = f"{r['sidecar_rank']}" if r.get("sidecar_rank") else "miss"
        ov = r.get("sidecar_top3_overlap", 0)
        q_short = r["query"][:54]
        print(f"{q_short:<55} {b_str:>7} {s_str:>7} {ov:>7.2f}")


if __name__ == "__main__":
    main()