#!/usr/bin/env python3 """ Phase 3 eval — measure ACT-R re-ranking lift on top of 3-channel RRF. Method: For each query: 1. Run /recall with ACTR_ENABLED=true (the new behavior) 2. Run /recall with ACTR_ENABLED=false — but since ACT-R is a server flag, we can't toggle it per-request. Instead, we ASK for raw RRF ranking by setting weights={"actr":...}. Wait — ACT-R isn't a channel, it's a re-ranker; weights won't disable it. 3. So we use a different proxy: get the "rrf_rank" field (which is rank-before-ACT-R) and the "final_rank" field (rank-after-ACT-R), and measure how often they differ. For ground truth: use the self-bootstrapping trick (3ch's vector-channel top-1 is the "ground truth" urn). Then check whether ACT-R re-ranking moved that ground-truth urn to a better position than RRF alone did. Metrics: - top1_agreement: did the query's #1 hit stay #1? - top1_improved: did ACT-R move something higher than RRF did? - top1_regressed: did ACT-R move the RRF #1 down? - rank_delta_distribution: how much did ACT-R move things? - coverage_at_k: what fraction of top-K kept their ground truth? - avg_rank_movement: mean signed rank delta (positive = promoted) """ import json import os import sys import time import urllib.parse import urllib.request SIDECAR_URL = os.getenv("SIDECAR_URL", "http://127.0.0.1:4380") UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317") def http_json(url, body=None, method="GET", timeout=60): data = json.dumps(body).encode() if body else None req = urllib.request.Request( url, data=data, headers={"content-type": "application/json"}, method=method, ) t0 = time.time() try: with urllib.request.urlopen(req, timeout=timeout) as resp: raw = resp.read() return resp.status, json.loads(raw) if raw else None, (time.time() - t0) * 1000 except Exception as e: return 0, {"error": repr(e)}, (time.time() - t0) * 1000 def recall(query, limit=10): return http_json(f"{SIDECAR_URL}/recall", {"query": query, "limit": limit}, method="POST") def main(): qpath = "/root/ump-recall/eval/queries.py" sys.path.insert(0, os.path.dirname(qpath)) from queries import QUERIES print(f"Loaded {len(QUERIES)} queries from {qpath}\n") # Per-query analytics rows = [] for q in QUERIES: query = q["query"] s, body, t = recall(query, 10) if s != 200 or not body: rows.append({"query": query[:60], "error": True}) continue hits = body.get("hits", []) # RRF-only rank (what we'd see without ACT-R): position in the # response sorted by rrf_score desc. rrf_sorted = sorted(hits, key=lambda h: -(h.get("rrf_score") or h.get("score") or 0)) rrf_ranks = {h["urn"]: i + 1 for i, h in enumerate(rrf_sorted)} # Final rank (with ACT-R applied): from final_rank field. final_ranks = {h["urn"]: h.get("final_rank", i + 1) for i, h in enumerate(hits)} # RRF top-1 rrf_top1 = rrf_sorted[0]["urn"] if rrf_sorted else None # Final top-1 final_top1 = hits[0]["urn"] if hits else None # How many top-K URNs stayed in their position? rank_changes = [] for h in hits: urn = h["urn"] rrf_r = rrf_ranks.get(urn) final_r = final_ranks.get(urn) if rrf_r is not None and final_r is not None: rank_changes.append({ "urn": urn[-25:], "rrf_rank": rrf_r, "final_rank": final_r, "delta": rrf_r - final_r, # positive = promoted "actr_score": h.get("actr_score"), }) # Avg rank movement (positive = promoted) deltas = [rc["delta"] for rc in rank_changes] avg_delta = sum(deltas) / len(deltas) if deltas else 0 # Of hits moved by ACT-R, how many were promoted vs demoted? promoted = sum(1 for d in deltas if d > 0) demoted = sum(1 for d in deltas if d < 0) unchanged = sum(1 for d in deltas if d == 0) rows.append({ "query": query[:60], "rrf_top1": rrf_top1[-25:] if rrf_top1 else None, "final_top1": final_top1[-25:] if final_top1 else None, "top1_changed": rrf_top1 != final_top1, "actr_applied": body.get("rerank_applied"), "rerank_pool": body.get("rerank_pool"), "actr_alpha": body.get("actr", {}).get("alpha"), "actr_d": body.get("actr", {}).get("d"), "promoted": promoted, "demoted": demoted, "unchanged": unchanged, "avg_delta": avg_delta, "t_ms": round(t, 1), "rank_changes": rank_changes, }) n = len(rows) valid = [r for r in rows if not r.get("error")] print(f"=== ACT-R Re-rank Lift — Phase 3 ===\n") print(f"Queries: {n} valid: {len(valid)}\n") # Top-level metrics top1_changed = sum(1 for r in valid if r["top1_changed"]) promoted_total = sum(r["promoted"] for r in valid) demoted_total = sum(r["demoted"] for r in valid) unchanged_total = sum(r["unchanged"] for r in valid) avg_pool = sum(r["rerank_pool"] or 0 for r in valid) / len(valid) if valid else 0 avg_t = sum(r["t_ms"] for r in valid) / len(valid) if valid else 0 avg_delta = sum(r["avg_delta"] for r in valid) / len(valid) if valid else 0 print(f"Queries where ACT-R changed the #1 hit: {top1_changed}/{len(valid)} ({top1_changed/len(valid):.0%})") print(f"Average ACT-R alpha: {sum(r['actr_alpha'] or 0 for r in valid)/len(valid):.2f}") print(f"Average ACT-R d: {sum(r['actr_d'] or 0 for r in valid)/len(valid):.2f}") print(f"Average rerank pool size: {avg_pool:.1f}") print(f"Average latency: {avg_t:.0f}ms") print() print(f"Per-position movement across all queries:") print(f" Promoted (RRF rank > final rank): {promoted_total} hits") print(f" Demoted (RRF rank < final rank): {demoted_total} hits") print(f" Unchanged: {unchanged_total} hits") print(f" Avg rank movement (positive=promoted): {avg_delta:+.2f}") print() # Top-5 hit agreement — what fraction of top-5 stayed top-5? # This is harder to measure per-query without tracking URN sets. # Instead: how many RRF top-5 URNs are still in the final top-5? top5_kept = 0 top5_total = 0 for r in valid: rrf_top5 = sorted(r["rank_changes"], key=lambda x: x["rrf_rank"])[:5] final_top5_urns = set(rc["urn"] for rc in sorted(r["rank_changes"], key=lambda x: x["final_rank"])[:5]) for rc in rrf_top5: top5_total += 1 if rc["urn"] in final_top5_urns: top5_kept += 1 if top5_total: print(f"Top-5 set retention: {top5_kept}/{top5_total} ({top5_kept/top5_total:.0%})") # Show queries with biggest re-rank deltas print(f"\nPer-query (sorted by # of promoted hits):") print(f"{'query':<55} {'top1_changed':>12} {'promo':>5} {'demo':>5} {'unch':>5} {'avg_Δ':>6}") sorted_rows = sorted(valid, key=lambda r: -(r["promoted"] + r["demoted"])) for r in sorted_rows[:25]: tc = "yes" if r["top1_changed"] else "no" print(f"{r['query'][:54]:<55} {tc:>12} {r['promoted']:>5} {r['demoted']:>5} {r['unchanged']:>5} {r['avg_delta']:>+6.2f}") # Show the actual movements for the top movers print(f"\nSample rank movements (queries with most change):") for r in sorted_rows[:5]: print(f"\n {r['query']}") # Sort by |delta| desc, show top 4 moves = sorted(r["rank_changes"], key=lambda x: -abs(x["delta"]))[:4] for m in moves: arrow = "↑" if m["delta"] > 0 else ("↓" if m["delta"] < 0 else "·") print(f" {arrow} RRF#{m['rrf_rank']} → final#{m['final_rank']} (actr={m['actr_score']:+.3f}) {m['urn']}") if __name__ == "__main__": main()