self-directed-learning/scripts/eval_channel_contribution.py

#!/usr/bin/env python3
"""
Phase 2 eval — Channel-contribution analysis.

Uses self-bootstrapping: for each query, the ground truth is the top-1
sidecar hit. Then we ask: which channels contributed to the top-K, and
did adding graph change the ranking?

Reports:
  - For each query, what fraction of top-5 came from each channel?
  - How often does graph contribute a UNIQUE hit (not in 2ch top-K)?
  - Average rank position of graph-only contributions
  - Latency cost of graph channel

This is more robust than the expected_id-based eval because it measures
the actual contribution of each channel rather than guessing what the
"right" answer is.
"""
import json
import os
import sys
import time
import urllib.request

SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380")


def http_json(url, body=None, method="GET", timeout=60):
    data = json.dumps(body).encode() if body else None
    req = urllib.request.Request(
        url, data=data,
        headers={"content-type": "application/json"},
        method=method,
    )
    t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
            return resp.status, json.loads(raw) if raw else None, (time.time() - t0) * 1000
    except Exception as e:
        return 0, {"error": repr(e)}, (time.time() - t0) * 1000


def recall(query, limit=10, with_graph=True):
    weights = None if with_graph else {"ump": 1.0, "vector": 1.0, "graph": 0.0}
    body = {"query": query, "limit": limit}
    if weights:
        body["weights"] = weights
    return http_json(f"{SIDECAR_URL}/recall", body, method="POST")


def channel_source(hit, mode="2ch"):
    """Which channel(s) contributed this hit. Returns set of channel names."""
    by_ch = hit.get("by_channel", {})
    if mode == "2ch":
        return set(k for k in by_ch if k in ("ump", "vector"))
    return set(by_ch.keys())


def main():
    qpath = "/root/ump-recall/eval/queries.py"
    sys.path.insert(0, os.path.dirname(qpath))
    from queries import QUERIES
    print(f"Loaded {len(QUERIES)} queries from {qpath}\n")

    rows = []
    for q in QUERIES:
        query = q["query"]

        s3, b3, t3 = recall(query, 10, with_graph=True)
        s2, b2, t2 = recall(query, 10, with_graph=False)

        if s3 != 200 or s2 != 200:
            rows.append({"query": query[:60], "error": True})
            continue

        hits3 = b3.get("hits", [])
        hits2 = b2.get("hits", [])

        # How many of 3ch's top-5 came via graph (graph-only or graph+other)?
        top5_3 = hits3[:5]
        graph_contrib_top5_3 = sum(1 for h in top5_3 if "graph" in h.get("by_channel", {}))

        top5_2 = hits2[:5]
        graph_contrib_top5_2 = sum(1 for h in top5_2 if "graph" in h.get("by_channel", {}))

        # Graph-unique: hits in 3ch top-5 that are NOT in 2ch top-5
        urns_3 = {h["urn"] for h in top5_3}
        urns_2 = {h["urn"] for h in top5_2}
        unique_to_3 = urns_3 - urns_2

        # For each graph-only hit, what was its rank in 3ch?
        graph_only_ranks = [
            i + 1 for i, h in enumerate(top5_3)
            if h["urn"] in unique_to_3 and "graph" in h.get("by_channel", {})
        ]

        rows.append({
            "query": query[:60],
            "graph_contrib_top5_3ch": graph_contrib_top5_3,
            "graph_contrib_top5_2ch": graph_contrib_top5_2,
            "unique_to_3ch_count": len(unique_to_3),
            "graph_only_ranks_in_3ch": graph_only_ranks,
            "t_2ch_ms": round(t2, 1),
            "t_3ch_ms": round(t3, 1),
        })

    n = len(rows)
    valid = [r for r in rows if not r.get("error")]
    print(f"Valid: {len(valid)}/{n}\n")

    if not valid:
        return

    # Per-query: graph channel contribution
    g_in_top5_3 = sum(r["graph_contrib_top5_3ch"] for r in valid) / len(valid)
    g_in_top5_2 = sum(r["graph_contrib_top5_2ch"] for r in valid) / len(valid)
    print(f"Avg graph-channel hits in top-5:")
    print(f"  3ch (graph enabled):    {g_in_top5_3:.2f} hits/query")
    print(f"  2ch (graph zero-weigh): {g_in_top5_2:.2f} hits/query")
    print(f"  Delta:                  {g_in_top5_3 - g_in_top5_2:+.2f} (should be ~equal since weights zero it)")

    # Unique-to-3ch count
    avg_unique = sum(r["unique_to_3ch_count"] for r in valid) / len(valid)
    total_unique = sum(r["unique_to_3ch_count"] for r in valid)
    print(f"\nHits in 3ch top-5 that are NOT in 2ch top-5:")
    print(f"  Total unique: {total_unique} (avg {avg_unique:.2f}/query)")

    # Latency
    avg_t2 = sum(r["t_2ch_ms"] for r in valid) / len(valid)
    avg_t3 = sum(r["t_3ch_ms"] for r in valid) / len(valid)
    print(f"\nLatency: 2ch={avg_t2:.0f}ms  3ch={avg_t3:.0f}ms  Δ={avg_t3-avg_t2:+.0f}ms (graph overhead)")

    # Show queries where graph added value
    print(f"\nPer-query graph contribution (top 20 by graph involvement):")
    print(f"{'query':<55} {'g-top5':>6} {'uniq':>5} {'2ch-ms':>7} {'3ch-ms':>7} {'g-ranks':>10}")
    sorted_rows = sorted(valid, key=lambda r: -r["graph_contrib_top5_3ch"])
    for r in sorted_rows[:20]:
        print(f"{r['query'][:54]:<55} {r['graph_contrib_top5_3ch']:>6} "
              f"{r['unique_to_3ch_count']:>5} {r['t_2ch_ms']:>7.0f} {r['t_3ch_ms']:>7.0f} "
              f"{str(r['graph_only_ranks_in_3ch']):>10}")


if __name__ == "__main__":
    main()