self-directed-learning/scripts/eval_actr.py

#!/usr/bin/env python3
"""
Phase 3 eval — measure ACT-R re-ranking lift on top of 3-channel RRF.

Method:
For each query:
  1. Run /recall with ACTR_ENABLED=true (the new behavior)
  2. Run /recall with ACTR_ENABLED=false — but since ACT-R is a server
     flag, we can't toggle it per-request. Instead, we ASK for raw RRF
     ranking by setting weights={"actr":...}. Wait — ACT-R isn't a
     channel, it's a re-ranker; weights won't disable it.
  3. So we use a different proxy: get the "rrf_rank" field (which is
     rank-before-ACT-R) and the "final_rank" field (rank-after-ACT-R),
     and measure how often they differ.

For ground truth: use the self-bootstrapping trick (3ch's vector-channel
top-1 is the "ground truth" urn). Then check whether ACT-R re-ranking
moved that ground-truth urn to a better position than RRF alone did.

Metrics:
  - top1_agreement: did the query's #1 hit stay #1?
  - top1_improved: did ACT-R move something higher than RRF did?
  - top1_regressed: did ACT-R move the RRF #1 down?
  - rank_delta_distribution: how much did ACT-R move things?
  - coverage_at_k: what fraction of top-K kept their ground truth?
  - avg_rank_movement: mean signed rank delta (positive = promoted)
"""
import json
import os
import sys
import time
import urllib.parse
import urllib.request

SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380")
UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317")


def http_json(url, body=None, method="GET", timeout=60):
    data = json.dumps(body).encode() if body else None
    req = urllib.request.Request(
        url, data=data,
        headers={"content-type": "application/json"},
        method=method,
    )
    t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read()
            return resp.status, json.loads(raw) if raw else None, (time.time() - t0) * 1000
    except Exception as e:
        return 0, {"error": repr(e)}, (time.time() - t0) * 1000


def recall(query, limit=10):
    return http_json(f"{SIDECAR_URL}/recall", {"query": query, "limit": limit}, method="POST")


def main():
    qpath = "/root/ump-recall/eval/queries.py"
    sys.path.insert(0, os.path.dirname(qpath))
    from queries import QUERIES
    print(f"Loaded {len(QUERIES)} queries from {qpath}\n")

    # Per-query analytics
    rows = []
    for q in QUERIES:
        query = q["query"]

        s, body, t = recall(query, 10)
        if s != 200 or not body:
            rows.append({"query": query[:60], "error": True})
            continue

        hits = body.get("hits", [])

        # RRF-only rank (what we'd see without ACT-R): position in the
        # response sorted by rrf_score desc.
        rrf_sorted = sorted(hits, key=lambda h: -(h.get("rrf_score") or h.get("score") or 0))
        rrf_ranks = {h["urn"]: i + 1 for i, h in enumerate(rrf_sorted)}

        # Final rank (with ACT-R applied): from final_rank field.
        final_ranks = {h["urn"]: h.get("final_rank", i + 1) for i, h in enumerate(hits)}

        # RRF top-1
        rrf_top1 = rrf_sorted[0]["urn"] if rrf_sorted else None
        # Final top-1
        final_top1 = hits[0]["urn"] if hits else None

        # How many top-K URNs stayed in their position?
        rank_changes = []
        for h in hits:
            urn = h["urn"]
            rrf_r = rrf_ranks.get(urn)
            final_r = final_ranks.get(urn)
            if rrf_r is not None and final_r is not None:
                rank_changes.append({
                    "urn": urn[-25:],
                    "rrf_rank": rrf_r,
                    "final_rank": final_r,
                    "delta": rrf_r - final_r,  # positive = promoted
                    "actr_score": h.get("actr_score"),
                })

        # Avg rank movement (positive = promoted)
        deltas = [rc["delta"] for rc in rank_changes]
        avg_delta = sum(deltas) / len(deltas) if deltas else 0

        # Of hits moved by ACT-R, how many were promoted vs demoted?
        promoted = sum(1 for d in deltas if d > 0)
        demoted = sum(1 for d in deltas if d < 0)
        unchanged = sum(1 for d in deltas if d == 0)

        rows.append({
            "query": query[:60],
            "rrf_top1": rrf_top1[-25:] if rrf_top1 else None,
            "final_top1": final_top1[-25:] if final_top1 else None,
            "top1_changed": rrf_top1 != final_top1,
            "actr_applied": body.get("rerank_applied"),
            "rerank_pool": body.get("rerank_pool"),
            "actr_alpha": body.get("actr", {}).get("alpha"),
            "actr_d": body.get("actr", {}).get("d"),
            "promoted": promoted,
            "demoted": demoted,
            "unchanged": unchanged,
            "avg_delta": avg_delta,
            "t_ms": round(t, 1),
            "rank_changes": rank_changes,
        })

    n = len(rows)
    valid = [r for r in rows if not r.get("error")]

    print(f"=== ACT-R Re-rank Lift — Phase 3 ===\n")
    print(f"Queries: {n}  valid: {len(valid)}\n")

    # Top-level metrics
    top1_changed = sum(1 for r in valid if r["top1_changed"])
    promoted_total = sum(r["promoted"] for r in valid)
    demoted_total = sum(r["demoted"] for r in valid)
    unchanged_total = sum(r["unchanged"] for r in valid)
    avg_pool = sum(r["rerank_pool"] or 0 for r in valid) / len(valid) if valid else 0
    avg_t = sum(r["t_ms"] for r in valid) / len(valid) if valid else 0
    avg_delta = sum(r["avg_delta"] for r in valid) / len(valid) if valid else 0

    print(f"Queries where ACT-R changed the #1 hit:    {top1_changed}/{len(valid)} ({top1_changed/len(valid):.0%})")
    print(f"Average ACT-R alpha:                       {sum(r['actr_alpha'] or 0 for r in valid)/len(valid):.2f}")
    print(f"Average ACT-R d:                           {sum(r['actr_d'] or 0 for r in valid)/len(valid):.2f}")
    print(f"Average rerank pool size:                  {avg_pool:.1f}")
    print(f"Average latency:                           {avg_t:.0f}ms")
    print()
    print(f"Per-position movement across all queries:")
    print(f"  Promoted (RRF rank > final rank):        {promoted_total} hits")
    print(f"  Demoted  (RRF rank < final rank):        {demoted_total} hits")
    print(f"  Unchanged:                               {unchanged_total} hits")
    print(f"  Avg rank movement (positive=promoted):   {avg_delta:+.2f}")
    print()

    # Top-5 hit agreement — what fraction of top-5 stayed top-5?
    # This is harder to measure per-query without tracking URN sets.
    # Instead: how many RRF top-5 URNs are still in the final top-5?
    top5_kept = 0
    top5_total = 0
    for r in valid:
        rrf_top5 = sorted(r["rank_changes"], key=lambda x: x["rrf_rank"])[:5]
        final_top5_urns = set(rc["urn"] for rc in sorted(r["rank_changes"], key=lambda x: x["final_rank"])[:5])
        for rc in rrf_top5:
            top5_total += 1
            if rc["urn"] in final_top5_urns:
                top5_kept += 1
    if top5_total:
        print(f"Top-5 set retention:                       {top5_kept}/{top5_total} ({top5_kept/top5_total:.0%})")

    # Show queries with biggest re-rank deltas
    print(f"\nPer-query (sorted by # of promoted hits):")
    print(f"{'query':<55} {'top1_changed':>12} {'promo':>5} {'demo':>5} {'unch':>5} {'avg_Δ':>6}")
    sorted_rows = sorted(valid, key=lambda r: -(r["promoted"] + r["demoted"]))
    for r in sorted_rows[:25]:
        tc = "yes" if r["top1_changed"] else "no"
        print(f"{r['query'][:54]:<55} {tc:>12} {r['promoted']:>5} {r['demoted']:>5} {r['unchanged']:>5} {r['avg_delta']:>+6.2f}")

    # Show the actual movements for the top movers
    print(f"\nSample rank movements (queries with most change):")
    for r in sorted_rows[:5]:
        print(f"\n  {r['query']}")
        # Sort by |delta| desc, show top 4
        moves = sorted(r["rank_changes"], key=lambda x: -abs(x["delta"]))[:4]
        for m in moves:
            arrow = "↑" if m["delta"] > 0 else ("↓" if m["delta"] < 0 else "·")
            print(f"    {arrow} RRF#{m['rrf_rank']} → final#{m['final_rank']} (actr={m['actr_score']:+.3f}) {m['urn']}")


if __name__ == "__main__":
    main()