Krystie
dc5dc94d79
Multi-channel retrieval sidecar over Universal Memory Protocol: - 3-channel RRF (UMP FTS5 + Qdrant vector + knowledge graph) - ACT-R re-ranking (Anderson 1983) with access tracking - Co-occurrence graph edges (Phase 6) for dense traversal - Memory lifecycle decay (Phase 4) with per-kind confidence - MCP shim routes recall through sidecar, falls back to canonical UMP Architecture: - src/server.js HTTP sidecar on port 4380 - src/graph.js 2592-node / 111-edge graph from UMP (or +cooccur: 13k+) - src/actr.js A_i = -d*ln(age) + beta*log1p(freq) + epsilon*conf - src/access_log.js per-URN counter + last_accessed_at - src/ump-recall-mcp.js MCP shim (recall via sidecar, others passthrough) Eval results (851-record UMP corpus): - 2ch RRF over baseline: +50pp recall@10 - 3ch RRF (+graph): +60pp, 12 unique wins - ACT-R re-rank: 4/20 #1 changes, 84% top-5 retention Tests: 76/76 passing across graph (27), actr (27), access_log (28), decay (20), mcp-shim (sidecar + fallback). Run with: npm test Inspired by AIAppsAPI/adaptive-recall but built from scratch against existing DNS2 infrastructure (UMP at :4317, Qdrant at :6333, Ollama at :11434). No paid SaaS, MIT-licensed.
103 lines
3.3 KiB
Python
103 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Phase 1F eval corpus auto-builder.
|
|
|
|
For each candidate query:
|
|
1. Verify the candidate expected_id actually exists via GET /ump/memory/<urn>
|
|
2. Verify the body.subject contains keywords from the query (loose match)
|
|
3. Only keep queries where the expected_id is a verified ground truth.
|
|
|
|
Output: eval/queries_verified.py with QUERIES_VERIFIED list.
|
|
|
|
This fixes the eval-broken-expected-ids problem from the first eval run:
|
|
the hand-picked expected_id values were guesses, not verified against
|
|
the real UMP store. Many turned out to be wrong.
|
|
|
|
Usage:
|
|
python3 build_verified_queries.py
|
|
python3 build_verified_queries.py --dry-run # print stats without writing
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
|
|
UMP_URL = os.getenv("UMP_URL", "http://127.0.0.1:4317")
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "eval"))
|
|
from queries import QUERIES
|
|
|
|
|
|
def http_get(url, timeout=5):
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
|
return resp.status, json.loads(resp.read())
|
|
except Exception as e:
|
|
return 0, {"error": repr(e)}
|
|
|
|
|
|
def fetch_record(urn):
|
|
status, body = http_get(f"{UMP_URL}/ump/memory/{urllib.parse.quote(urn, safe='')}")
|
|
if status != 200:
|
|
return None
|
|
return body.get("record") or body
|
|
|
|
|
|
def query_contains_keywords(query, text, min_overlap=1):
|
|
"""Returns True if any query keyword appears in text."""
|
|
query_words = {w.lower() for w in query.split() if len(w) > 3}
|
|
text_lower = (text or "").lower()
|
|
hits = sum(1 for w in query_words if w in text_lower)
|
|
return hits >= min_overlap
|
|
|
|
|
|
def main():
|
|
verified = []
|
|
rejected = []
|
|
|
|
for q in QUERIES:
|
|
query = q["query"]
|
|
exp = q.get("expected_id")
|
|
if not exp:
|
|
rejected.append({"query": query, "reason": "no expected_id"})
|
|
continue
|
|
|
|
rec = fetch_record(exp)
|
|
if not rec:
|
|
rejected.append({"query": query, "expected_id": exp, "reason": "urn not found in UMP"})
|
|
continue
|
|
|
|
# Check subject contains at least one keyword from query
|
|
body = rec.get("body", {})
|
|
subject = body.get("subject", "")
|
|
text = body.get("text", "")
|
|
combined = f"{subject} {text}"
|
|
if not query_contains_keywords(query, combined, min_overlap=1):
|
|
rejected.append({
|
|
"query": query,
|
|
"expected_id": exp,
|
|
"reason": f"no keyword overlap; subject='{subject[:60]}'",
|
|
})
|
|
continue
|
|
|
|
verified.append(q)
|
|
|
|
print(f"Verified: {len(verified)} / {len(QUERIES)}")
|
|
print(f"Rejected: {len(rejected)}")
|
|
for r in rejected:
|
|
print(f" - {r['query'][:50]} | {r['reason']}")
|
|
|
|
# Write eval/queries_verified.py
|
|
out_path = os.path.join(os.path.dirname(__file__), "..", "eval", "queries_verified.py")
|
|
with open(out_path, "w") as f:
|
|
f.write("# Auto-verified eval queries. Regenerate with build_verified_queries.py\n")
|
|
f.write("# Each query's expected_id was confirmed to exist in UMP AND its body\n")
|
|
f.write("# shares at least one keyword with the query.\n\n")
|
|
f.write(f"QUERIES_VERIFIED = {json.dumps(verified, indent=2)}\n")
|
|
|
|
print(f"\nWrote {len(verified)} verified queries to {out_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import urllib.parse
|
|
main() |