skill-evolution/tests/test_evaluate_tool_calls_target.py

"""Tests for scripts/evaluate.py's evaluate_tool_calls() target (U2, R5)."""

import json
import os
import sys

sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))

import pytest

import evaluate
from evaluate import EvalResult, evaluate_tool_calls


@pytest.fixture(autouse=True)
def stub_run_evaluators(monkeypatch):
    captured = {}

    def fake_run_evaluators(content, target, context=None):
        captured["content"] = content
        captured["target"] = target
        captured["context"] = context or {}
        return [EvalResult(score=0.8, passed=True, feedback="ok", evaluator_name="stub")]

    monkeypatch.setattr(evaluate, "run_evaluators", fake_run_evaluators)
    return captured


def test_evaluate_tool_calls_extracts_tool_calls_from_messages(stub_run_evaluators):
    messages = [
        {
            "role": "assistant",
            "content": "I'll search for the file.",
            "tool_calls": [
                {"name": "read_file", "arguments": {"path": "/tmp/test.py"}, "result": "file contents"},
            ],
        },
    ]
    results = evaluate_tool_calls(messages)

    assert stub_run_evaluators["target"] == "tool_calls:unknown-session"
    assert "read_file" in stub_run_evaluators["content"]
    assert results[0].passed is True


def test_evaluate_tool_calls_with_session_id_queries_db(monkeypatch, stub_run_evaluators):
    """When given a session_id string, it should query state.db."""
    fake_messages = [
        {
            "role": "assistant",
            "content": "Searching...",
            "tool_calls": [
                {"name": "grep", "arguments": {"pattern": "test"}, "result": "found 3 matches"},
            ],
        },
    ]

    def fake_fetch(session_id):
        assert session_id == "test-session-123"
        return fake_messages

    monkeypatch.setattr(evaluate, "_fetch_session_messages", fake_fetch)

    results = evaluate_tool_calls("test-session-123")

    assert stub_run_evaluators["target"] == "tool_calls:test-session-123"
    assert "grep" in stub_run_evaluators["content"]
    assert results[0].passed is True


def test_evaluate_tool_calls_no_tool_calls_returns_passing(stub_run_evaluators):
    """Edge: a session with no tool_calls returns a passing 'no data' result."""
    messages = [
        {"role": "user", "content": "Hello"},
        {"role": "assistant", "content": "Hi there!"},
    ]
    results = evaluate_tool_calls(messages)

    assert "(no tool calls recorded" in stub_run_evaluators["content"]
    assert stub_run_evaluators["target"] == "tool_calls:unknown-session"
    assert results[0].passed is True


def test_evaluate_tool_calls_handles_malformed_json(stub_run_evaluators):
    """Edge: malformed tool_calls JSON is handled defensively."""
    messages = [
        {
            "role": "assistant",
            "content": "test",
            "tool_calls": "not valid json {{{",
        },
    ]
    results = evaluate_tool_calls(messages)

    assert results[0].passed is True


def test_evaluate_tool_calls_caps_result_snippets(stub_run_evaluators):
    """Result snippets are capped at 200 chars to control cost."""
    long_result = "x" * 500
    messages = [
        {
            "role": "assistant",
            "content": "test",
            "tool_calls": [
                {"name": "read_file", "arguments": {}, "result": long_result},
            ],
        },
    ]
    evaluate_tool_calls(messages)

    # The content should contain the truncated result
    assert "x" in stub_run_evaluators["content"]
    # ... but not the full 500 chars
    assert len(stub_run_evaluators["content"]) < 1000


def test_evaluate_tool_calls_does_not_modify_evaluate_skill_text(stub_run_evaluators):
    """KTD1: evaluate_tool_calls is a sibling, not a branch in evaluate_skill_text."""
    messages = [
        {
            "role": "assistant",
            "content": "test",
            "tool_calls": [{"name": "grep", "arguments": {}, "result": "ok"}],
        },
    ]
    evaluate_tool_calls(messages)
    assert stub_run_evaluators["target"].startswith("tool_calls:")