Initial import of NousResearch/hermes-agent
Deploy Site / deploy-vercel (push) Has been cancelled
Deploy Site / deploy-docs (push) Has been cancelled
Docker / shell lint / Lint Dockerfile (hadolint) (push) Has been cancelled
Docker / shell lint / Lint docker/ shell scripts (shellcheck) (push) Has been cancelled
Docker Build and Publish / build-amd64 (push) Has been cancelled
Docker Build and Publish / build-arm64 (push) Has been cancelled
Lint (ruff + ty) / ruff + ty diff (push) Has been cancelled
Lint (ruff + ty) / ruff enforcement (blocking) (push) Has been cancelled
Lint (ruff + ty) / Windows footguns (blocking) (push) Has been cancelled
Nix Lockfile Fix / auto-fix-main (push) Has been cancelled
Nix Lockfile Fix / fix (push) Has been cancelled
Nix / nix (macos-latest) (push) Has been cancelled
Nix / nix (ubuntu-latest) (push) Has been cancelled
OSV-Scanner / Scan lockfiles (push) Has been cancelled
Build Skills Index / build-index (push) Has been cancelled
Tests / test (1) (push) Has been cancelled
Tests / test (2) (push) Has been cancelled
Tests / test (3) (push) Has been cancelled
Tests / test (4) (push) Has been cancelled
Tests / test (5) (push) Has been cancelled
Tests / test (6) (push) Has been cancelled
Tests / e2e (push) Has been cancelled
uv.lock check / uv lock --check (push) Has been cancelled
Docker Build and Publish / merge (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
Tests / save-durations (push) Has been cancelled

This commit is contained in:
红尘
2026-05-31 09:36:58 +08:00
commit d73ff9b0fb
4188 changed files with 1614916 additions and 0 deletions
View File
View File
+243
View File
@@ -0,0 +1,243 @@
"""Tests for GHSA-96vc-wcxf-jjff and GHSA-qg5c-hvr5-hjgr.
Two related ACP approval-flow issues:
- 96vc: ACP didn't set HERMES_EXEC_ASK, so `check_all_command_guards`
took the non-interactive auto-approve path and never consulted the
ACP-supplied callback.
- qg5c: `_approval_callback` was a module-global in terminal_tool;
overlapping ACP sessions overwrote each other's callback slot.
Both fixed together by:
1. Setting HERMES_EXEC_ASK inside _run_agent (wraps the agent call).
2. Storing the callback in thread-local state so concurrent executor
threads don't collide.
"""
import threading
class TestThreadLocalApprovalCallback:
"""GHSA-qg5c-hvr5-hjgr: set_approval_callback must be per-thread so
concurrent ACP sessions don't stomp on each other's handlers."""
def test_set_and_get_in_same_thread(self):
from tools.terminal_tool import (
set_approval_callback,
_get_approval_callback,
)
cb1 = lambda cmd, desc: "once" # noqa: E731
set_approval_callback(cb1)
assert _get_approval_callback() is cb1
def test_callback_not_visible_in_different_thread(self):
"""Thread A's callback is NOT visible to Thread B."""
from tools.terminal_tool import (
set_approval_callback,
_get_approval_callback,
)
cb_a = lambda cmd, desc: "thread_a" # noqa: E731
cb_b = lambda cmd, desc: "thread_b" # noqa: E731
seen_in_a = []
seen_in_b = []
def thread_a():
set_approval_callback(cb_a)
# Pause so thread B has time to set its own callback
import time
time.sleep(0.05)
seen_in_a.append(_get_approval_callback())
def thread_b():
set_approval_callback(cb_b)
import time
time.sleep(0.05)
seen_in_b.append(_get_approval_callback())
ta = threading.Thread(target=thread_a)
tb = threading.Thread(target=thread_b)
ta.start()
tb.start()
ta.join()
tb.join()
# Each thread must see ONLY its own callback — not the other's
assert seen_in_a == [cb_a]
assert seen_in_b == [cb_b]
def test_main_thread_callback_not_leaked_to_worker(self):
"""A callback set in the main thread does NOT leak into a
freshly-spawned worker thread."""
from tools.terminal_tool import (
set_approval_callback,
_get_approval_callback,
)
cb_main = lambda cmd, desc: "main" # noqa: E731
set_approval_callback(cb_main)
worker_saw = []
def worker():
worker_saw.append(_get_approval_callback())
t = threading.Thread(target=worker)
t.start()
t.join()
# Worker thread has no callback set — TLS is empty for it
assert worker_saw == [None]
# Main thread still has its callback
assert _get_approval_callback() is cb_main
def test_sudo_password_callback_also_thread_local(self):
"""Same protection applies to the sudo password callback."""
from tools.terminal_tool import (
set_sudo_password_callback,
_get_sudo_password_callback,
)
cb_main = lambda: "main-password" # noqa: E731
set_sudo_password_callback(cb_main)
worker_saw = []
def worker():
worker_saw.append(_get_sudo_password_callback())
t = threading.Thread(target=worker)
t.start()
t.join()
assert worker_saw == [None]
assert _get_sudo_password_callback() is cb_main
def test_sudo_password_cache_does_not_leak_across_threads(self):
"""Interactive sudo cache must not bleed into another executor thread."""
from tools.terminal_tool import (
_get_cached_sudo_password,
_reset_cached_sudo_passwords,
_set_cached_sudo_password,
)
_reset_cached_sudo_passwords()
_set_cached_sudo_password("main-thread-password")
worker_saw = []
def worker():
worker_saw.append(_get_cached_sudo_password())
t = threading.Thread(target=worker)
t.start()
t.join()
assert worker_saw == [""]
assert _get_cached_sudo_password() == "main-thread-password"
def test_sudo_password_cache_isolated_across_acp_sessions_on_same_pool_thread(self):
"""ACP's ThreadPoolExecutor reuses threads. Two ACP sessions that land
on the same reused thread must not share the interactive sudo password
cache. The fix wraps each session in contextvars.copy_context() and
binds HERMES_SESSION_KEY per session, so the cache scope key differs
across sessions even when the underlying thread is identical.
"""
import contextvars
from concurrent.futures import ThreadPoolExecutor
from gateway.session_context import (
clear_session_vars,
set_session_vars,
)
from tools.terminal_tool import (
_get_cached_sudo_password,
_reset_cached_sudo_passwords,
_set_cached_sudo_password,
)
_reset_cached_sudo_passwords()
executor = ThreadPoolExecutor(max_workers=1) # force thread reuse
runs: list[tuple[str, str, str]] = [] # (session_id, before, after)
def _simulate_acp_session(session_id: str, write_password: str) -> None:
tokens = set_session_vars(session_key=session_id)
try:
observed_before = _get_cached_sudo_password()
_set_cached_sudo_password(write_password)
observed_after = _get_cached_sudo_password()
runs.append((session_id, observed_before, observed_after))
finally:
clear_session_vars(tokens)
def _run_in_fresh_context(session_id: str, pw: str) -> str:
ctx = contextvars.copy_context()
ctx.run(_simulate_acp_session, session_id, pw)
return session_id
try:
executor.submit(_run_in_fresh_context, "acp-session-A", "alpha-secret").result()
# Same thread. Without the fix B would see "alpha-secret".
executor.submit(_run_in_fresh_context, "acp-session-B", "bravo-secret").result()
finally:
executor.shutdown(wait=True)
_reset_cached_sudo_passwords()
assert runs[0] == ("acp-session-A", "", "alpha-secret")
# Core regression guard: B on the same reused thread must see an empty
# cache, not A's password.
assert runs[1] == ("acp-session-B", "", "bravo-secret")
class TestAcpExecAskGate:
"""GHSA-96vc-wcxf-jjff: ACP's _run_agent must set HERMES_INTERACTIVE so
that tools.approval.check_all_command_guards takes the CLI-interactive
path (consults the registered callback via prompt_dangerous_approval)
instead of the non-interactive auto-approve shortcut.
(HERMES_EXEC_ASK takes the gateway-queue path which requires a
notify_cb registered in _gateway_notify_cbs — not applicable to ACP,
which uses a direct callback shape.)"""
def test_interactive_env_var_routes_to_callback(self, monkeypatch):
"""When HERMES_INTERACTIVE is set and an approval callback is
registered, a dangerous command must route through the callback."""
# Clean env
monkeypatch.delenv("HERMES_INTERACTIVE", raising=False)
monkeypatch.delenv("HERMES_GATEWAY_SESSION", raising=False)
monkeypatch.delenv("HERMES_EXEC_ASK", raising=False)
monkeypatch.delenv("HERMES_YOLO_MODE", raising=False)
from tools.approval import check_all_command_guards
called_with = []
def fake_cb(command, description, *, allow_permanent=True):
called_with.append((command, description))
return "once"
# Without HERMES_INTERACTIVE: takes auto-approve path, callback NOT called
result = check_all_command_guards(
"rm -rf /tmp/test-exec-ask", "local", approval_callback=fake_cb,
)
assert result["approved"] is True
assert called_with == [], (
"without HERMES_INTERACTIVE the non-interactive auto-approve "
"path should fire without consulting the callback"
)
# With HERMES_INTERACTIVE: callback IS called, approval flows through it
monkeypatch.setenv("HERMES_INTERACTIVE", "1")
called_with.clear()
result = check_all_command_guards(
"rm -rf /tmp/test-exec-ask", "local", approval_callback=fake_cb,
)
assert called_with, (
"with HERMES_INTERACTIVE the approval path should consult the "
"registered callback — this was the ACP bypass in "
"GHSA-96vc-wcxf-jjff"
)
assert result["approved"] is True
+102
View File
@@ -0,0 +1,102 @@
"""Tests for acp_adapter.auth — provider detection."""
from acp_adapter.auth import (
TERMINAL_SETUP_AUTH_METHOD_ID,
build_auth_methods,
has_provider,
detect_provider,
)
class TestHasProvider:
def test_has_provider_with_resolved_runtime(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": "openrouter", "api_key": "sk-or-test"},
)
assert has_provider() is True
def test_has_no_provider_when_runtime_has_no_key(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": "openrouter", "api_key": ""},
)
assert has_provider() is False
def test_has_no_provider_when_runtime_resolution_fails(self, monkeypatch):
def _boom():
raise RuntimeError("no provider")
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _boom)
assert has_provider() is False
class TestDetectProvider:
def test_detect_openrouter(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": "openrouter", "api_key": "sk-or-test"},
)
assert detect_provider() == "openrouter"
def test_detect_anthropic(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": "anthropic", "api_key": "sk-ant-test"},
)
assert detect_provider() == "anthropic"
def test_detect_none_when_no_key(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": "kimi-coding", "api_key": ""},
)
assert detect_provider() is None
def test_detect_none_on_resolution_error(self, monkeypatch):
def _boom():
raise RuntimeError("broken")
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _boom)
assert detect_provider() is None
def test_detect_provider_strips_and_lowercases_provider(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": " OpenRouter ", "api_key": " sk-or-test "},
)
assert detect_provider() == "openrouter"
class TestBuildAuthMethods:
def test_build_auth_methods_returns_provider_and_terminal_when_configured(self, monkeypatch):
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: "openrouter")
methods = build_auth_methods()
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in methods]
assert payloads[0]["id"] == "openrouter"
assert payloads[0]["name"] == "openrouter runtime credentials"
assert any(payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID for payload in payloads)
terminal = next(payload for payload in payloads if payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID)
assert terminal["type"] == "terminal"
assert terminal["args"] == ["--setup"]
def test_build_auth_methods_returns_terminal_setup_when_unconfigured(self, monkeypatch):
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: None)
methods = build_auth_methods()
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in methods]
assert payloads == [
{
"args": ["--setup"],
"description": (
"Open Hermes' interactive model/provider setup in a terminal. "
"Use this when Hermes has not been configured on this machine yet."
),
"id": TERMINAL_SETUP_AUTH_METHOD_ID,
"name": "Configure Hermes provider",
"type": "terminal",
}
]
+207
View File
@@ -0,0 +1,207 @@
"""Tests for ACP pre-edit approval gating."""
from __future__ import annotations
import json
import tempfile
from pathlib import Path
from acp_adapter.edit_approval import (
EditProposal,
build_acp_edit_tool_call,
clear_edit_approval_requester,
set_edit_approval_requester,
should_auto_approve_edit,
)
from model_tools import handle_function_call
def teardown_function() -> None:
clear_edit_approval_requester()
def test_acp_permission_tool_call_uses_edit_kind_and_diff_content():
proposal = EditProposal(
tool_name="write_file",
path="demo.txt",
old_text="old\n",
new_text="new\n",
arguments={"path": "demo.txt", "content": "new\n"},
)
tool_call = build_acp_edit_tool_call(proposal)
assert tool_call.kind == "edit"
assert tool_call.status == "pending"
assert tool_call.rawInput == {"tool": "write_file", "arguments": proposal.arguments}
assert len(tool_call.content) == 1
diff = tool_call.content[0]
assert diff.path == "demo.txt"
assert diff.oldText == "old\n"
assert diff.newText == "new\n"
def test_write_file_rejection_does_not_mutate_existing_file(tmp_path):
target = tmp_path / "sample.txt"
target.write_text("before\n", encoding="utf-8")
set_edit_approval_requester(lambda _proposal: False)
result = json.loads(
handle_function_call(
"write_file",
{"path": str(target), "content": "after\n"},
task_id="acp-edit-reject",
)
)
assert "error" in result
assert "Edit approval denied" in result["error"]
assert target.read_text(encoding="utf-8") == "before\n"
def test_write_file_approval_mutates_and_request_includes_diff(tmp_path):
target = tmp_path / "sample.txt"
target.write_text("before\n", encoding="utf-8")
proposals = []
def approve(proposal):
proposals.append(proposal)
return True
set_edit_approval_requester(approve)
result = json.loads(
handle_function_call(
"write_file",
{"path": str(target), "content": "after\n"},
task_id="acp-edit-approve",
)
)
assert result.get("bytes_written") == len("after\n")
assert target.read_text(encoding="utf-8") == "after\n"
assert len(proposals) == 1
proposal = proposals[0]
assert proposal.tool_name == "write_file"
assert proposal.path == str(target)
assert proposal.old_text == "before\n"
assert proposal.new_text == "after\n"
def test_write_file_new_file_request_has_empty_old_text(tmp_path):
target = tmp_path / "new.txt"
proposals = []
set_edit_approval_requester(lambda proposal: proposals.append(proposal) or True)
result = json.loads(
handle_function_call(
"write_file",
{"path": str(target), "content": "created\n"},
task_id="acp-edit-new-file",
)
)
assert result.get("bytes_written") == len("created\n")
assert target.read_text(encoding="utf-8") == "created\n"
assert proposals[0].old_text is None
assert proposals[0].new_text == "created\n"
def test_requester_exception_denies_and_does_not_mutate(tmp_path):
target = tmp_path / "sample.txt"
target.write_text("before\n", encoding="utf-8")
def boom(_proposal):
raise RuntimeError("zed disconnected")
set_edit_approval_requester(boom)
result = json.loads(
handle_function_call(
"write_file",
{"path": str(target), "content": "after\n"},
task_id="acp-edit-exception",
)
)
assert "error" in result
assert "Edit approval denied" in result["error"]
assert target.read_text(encoding="utf-8") == "before\n"
def test_patch_replace_rejection_does_not_mutate(tmp_path):
target = tmp_path / "sample.txt"
target.write_text("alpha\nbeta\n", encoding="utf-8")
set_edit_approval_requester(lambda _proposal: False)
result = json.loads(
handle_function_call(
"patch",
{
"mode": "replace",
"path": str(target),
"old_string": "beta\n",
"new_string": "gamma\n",
},
task_id="acp-patch-reject",
)
)
assert "error" in result
assert "Edit approval denied" in result["error"]
assert target.read_text(encoding="utf-8") == "alpha\nbeta\n"
def test_patch_replace_approval_request_includes_full_file_diff(tmp_path):
target = tmp_path / "sample.txt"
target.write_text("alpha\nbeta\n", encoding="utf-8")
proposals = []
set_edit_approval_requester(lambda proposal: proposals.append(proposal) or True)
result = json.loads(
handle_function_call(
"patch",
{
"mode": "replace",
"path": str(target),
"old_string": "beta\n",
"new_string": "gamma\n",
},
task_id="acp-patch-approve",
)
)
assert result.get("success") is True
assert target.read_text(encoding="utf-8") == "alpha\ngamma\n"
assert proposals[0].tool_name == "patch"
assert proposals[0].old_text == "alpha\nbeta\n"
assert proposals[0].new_text == "alpha\ngamma\n"
def test_workspace_auto_approval_allows_workspace_and_tmp_but_not_sensitive(tmp_path):
workspace_file = tmp_path / "src.py"
# Use tempfile.gettempdir() so this test exercises the same code path on
# Linux (`/tmp`), macOS (`/private/var/folders/...`) and Windows
# (`%LOCALAPPDATA%\Temp`). Before the fix this branch only worked on Linux.
tmp_file = Path(tempfile.gettempdir()) / "hermes-acp-auto-approve-test.txt"
env_file = tmp_path / ".env"
assert should_auto_approve_edit(
EditProposal("write_file", str(workspace_file), None, "x", {}),
"workspace_session",
str(tmp_path),
)
assert should_auto_approve_edit(
EditProposal("write_file", str(tmp_file), None, "x", {}),
"workspace_session",
str(tmp_path),
)
assert not should_auto_approve_edit(
EditProposal("write_file", str(env_file), None, "SECRET=x", {}),
"session",
str(tmp_path),
)
+155
View File
@@ -0,0 +1,155 @@
"""Tests for acp_adapter.entry startup wiring."""
import sys
import acp
import pytest
from acp_adapter import entry
def test_main_enables_unstable_protocol(monkeypatch):
calls = {}
async def fake_run_agent(agent, **kwargs):
calls["kwargs"] = kwargs
monkeypatch.setattr(entry, "_setup_logging", lambda: None)
monkeypatch.setattr(entry, "_load_env", lambda: None)
monkeypatch.setattr(acp, "run_agent", fake_run_agent)
entry.main([])
assert calls["kwargs"]["use_unstable_protocol"] is True
def test_main_version_prints_without_starting_server(monkeypatch, capsys):
monkeypatch.setattr(entry, "_setup_logging", lambda: (_ for _ in ()).throw(AssertionError("started server")))
entry.main(["--version"])
output = capsys.readouterr().out.strip()
assert output
assert "Starting hermes-agent ACP adapter" not in output
def test_main_check_prints_ok_without_starting_server(monkeypatch, capsys):
monkeypatch.setattr(entry, "_setup_logging", lambda: (_ for _ in ()).throw(AssertionError("started server")))
entry.main(["--check"])
assert capsys.readouterr().out.strip() == "Hermes ACP check OK"
def test_main_setup_runs_model_configuration(monkeypatch):
calls = {}
def fake_hermes_main():
calls["argv"] = sys.argv[:]
monkeypatch.setattr("hermes_cli.main.main", fake_hermes_main)
# Pretend stdin is not a TTY so the follow-up browser prompt is skipped.
# That keeps this test focused on the model-setup wiring; the
# browser-prompt path has its own test below.
monkeypatch.setattr("sys.stdin.isatty", lambda: False)
entry.main(["--setup"])
assert calls["argv"][1:] == ["model"]
def test_main_setup_offers_browser_install_when_tty(monkeypatch):
"""When stdin is a TTY and the user answers yes, model setup is followed
by a browser-tools bootstrap call."""
monkeypatch.setattr("hermes_cli.main.main", lambda: None)
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
monkeypatch.setattr("builtins.input", lambda *_args, **_kwargs: "y")
bootstrap_calls = []
monkeypatch.setattr(
entry,
"_run_setup_browser",
lambda assume_yes=False: bootstrap_calls.append(assume_yes) or 0,
)
entry.main(["--setup"])
assert bootstrap_calls == [False]
def test_main_setup_skips_browser_prompt_on_no(monkeypatch):
monkeypatch.setattr("hermes_cli.main.main", lambda: None)
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
monkeypatch.setattr("builtins.input", lambda *_args, **_kwargs: "")
called = []
monkeypatch.setattr(
entry,
"_run_setup_browser",
lambda assume_yes=False: called.append(assume_yes) or 0,
)
entry.main(["--setup"])
assert called == []
def test_main_setup_browser_calls_ensure_dependency(monkeypatch):
"""`hermes-acp --setup-browser` routes through dep_ensure.ensure_dependency."""
calls = []
def fake_ensure(dep, interactive=True):
calls.append((dep, interactive))
return True
monkeypatch.setattr("hermes_cli.dep_ensure.ensure_dependency", fake_ensure)
entry.main(["--setup-browser"])
assert ("node", True) in calls
assert ("browser", True) in calls
def test_main_setup_browser_forwards_yes_flag(monkeypatch):
"""--yes suppresses interactive prompts in ensure_dependency."""
calls = []
def fake_ensure(dep, interactive=True):
calls.append((dep, interactive))
return True
monkeypatch.setattr("hermes_cli.dep_ensure.ensure_dependency", fake_ensure)
entry.main(["--setup-browser", "--yes"])
assert ("node", False) in calls
assert ("browser", False) in calls
def test_main_setup_browser_stops_on_node_failure(monkeypatch):
"""If node install fails, browser install is not attempted."""
calls = []
def fake_ensure(dep, interactive=True):
calls.append(dep)
return dep != "node" # node fails
monkeypatch.setattr("hermes_cli.dep_ensure.ensure_dependency", fake_ensure)
with pytest.raises(SystemExit) as excinfo:
entry.main(["--setup-browser"])
assert excinfo.value.code == 1
assert "node" in calls
assert "browser" not in calls
def test_main_setup_browser_propagates_browser_failure(monkeypatch):
"""If browser install fails, exit code is 1."""
def fake_ensure(dep, interactive=True):
return dep != "browser" # browser fails
monkeypatch.setattr("hermes_cli.dep_ensure.ensure_dependency", fake_ensure)
with pytest.raises(SystemExit) as excinfo:
entry.main(["--setup-browser"])
assert excinfo.value.code == 1
+422
View File
@@ -0,0 +1,422 @@
"""Tests for acp_adapter.events — callback factories for ACP notifications."""
import asyncio
import gc
import warnings
from concurrent.futures import Future
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import acp
from acp.schema import AgentPlanUpdate
from acp_adapter.events import (
_build_plan_update_from_todo_result,
_send_update,
make_message_cb,
make_step_cb,
make_thinking_cb,
make_tool_progress_cb,
)
@pytest.fixture()
def mock_conn():
"""Mock ACP Client connection."""
conn = MagicMock(spec=acp.Client)
conn.session_update = AsyncMock()
return conn
@pytest.fixture()
def event_loop_fixture():
"""Create a real event loop for testing threadsafe coroutine submission."""
loop = asyncio.new_event_loop()
yield loop
loop.close()
# ---------------------------------------------------------------------------
# Tool progress callback
# ---------------------------------------------------------------------------
class TestToolProgressCallback:
def test_emits_tool_call_start(self, mock_conn, event_loop_fixture):
"""Tool progress should emit a ToolCallStart update."""
tool_call_ids = {}
tool_call_meta = {}
loop = event_loop_fixture
cb = make_tool_progress_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
# Run callback in the event loop context
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb("tool.started", "terminal", "$ ls -la", {"command": "ls -la"})
# Should have tracked the tool call ID
assert "terminal" in tool_call_ids
# Should have called run_coroutine_threadsafe
mock_rcts.assert_called_once()
coro = mock_rcts.call_args[0][0]
# The coroutine should be conn.session_update
assert mock_conn.session_update.called or coro is not None
def test_handles_string_args(self, mock_conn, event_loop_fixture):
"""If args is a JSON string, it should be parsed."""
tool_call_ids = {}
tool_call_meta = {}
loop = event_loop_fixture
cb = make_tool_progress_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb("tool.started", "read_file", "Reading /etc/hosts", '{"path": "/etc/hosts"}')
assert "read_file" in tool_call_ids
def test_handles_non_dict_args(self, mock_conn, event_loop_fixture):
"""If args is not a dict, it should be wrapped."""
tool_call_ids = {}
tool_call_meta = {}
loop = event_loop_fixture
cb = make_tool_progress_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb("tool.started", "terminal", "$ echo hi", None)
assert "terminal" in tool_call_ids
def test_duplicate_same_name_tool_calls_use_fifo_ids(self, mock_conn, event_loop_fixture):
"""Multiple same-name tool calls should be tracked independently in order."""
tool_call_ids = {}
tool_call_meta = {}
loop = event_loop_fixture
progress_cb = make_tool_progress_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
step_cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
progress_cb("tool.started", "terminal", "$ ls", {"command": "ls"})
progress_cb("tool.started", "terminal", "$ pwd", {"command": "pwd"})
assert len(tool_call_ids["terminal"]) == 2
step_cb(1, [{"name": "terminal", "result": "ok-1"}])
assert len(tool_call_ids["terminal"]) == 1
step_cb(2, [{"name": "terminal", "result": "ok-2"}])
assert "terminal" not in tool_call_ids
# ---------------------------------------------------------------------------
# Thinking callback
# ---------------------------------------------------------------------------
class TestThinkingCallback:
def test_emits_thought_chunk(self, mock_conn, event_loop_fixture):
"""Thinking callback should emit AgentThoughtChunk."""
loop = event_loop_fixture
cb = make_thinking_cb(mock_conn, "session-1", loop)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb("Analyzing the code...")
mock_rcts.assert_called_once()
def test_ignores_empty_text(self, mock_conn, event_loop_fixture):
"""Empty text should not emit any update."""
loop = event_loop_fixture
cb = make_thinking_cb(mock_conn, "session-1", loop)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
cb("")
mock_rcts.assert_not_called()
# ---------------------------------------------------------------------------
# Step callback
# ---------------------------------------------------------------------------
class TestStepCallback:
def test_completes_tracked_tool_calls(self, mock_conn, event_loop_fixture):
"""Step callback should mark tracked tools as completed."""
tool_call_ids = {"terminal": "tc-abc123"}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, {})
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb(1, [{"name": "terminal", "result": "success"}])
# Tool should have been removed from tracking
assert "terminal" not in tool_call_ids
mock_rcts.assert_called_once()
def test_ignores_untracked_tools(self, mock_conn, event_loop_fixture):
"""Tools not in tool_call_ids should be silently ignored."""
tool_call_ids = {}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, {})
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
cb(1, [{"name": "unknown_tool", "result": "ok"}])
mock_rcts.assert_not_called()
def test_handles_string_tool_info(self, mock_conn, event_loop_fixture):
"""Tool info as a string (just the name) should work."""
tool_call_ids = {"read_file": "tc-def456"}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, {})
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb(2, ["read_file"])
assert "read_file" not in tool_call_ids
mock_rcts.assert_called_once()
def test_result_passed_to_build_tool_complete(self, mock_conn, event_loop_fixture):
"""Tool result from prev_tools dict is forwarded to build_tool_complete."""
from collections import deque
tool_call_ids = {"terminal": deque(["tc-xyz789"])}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, {})
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts, \
patch("acp_adapter.events.build_tool_complete") as mock_btc:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
# Provide a result string in the tool info dict
cb(1, [{"name": "terminal", "result": '{"output": "hello"}'}])
mock_btc.assert_called_once_with(
"tc-xyz789", "terminal", result='{"output": "hello"}', function_args=None, snapshot=None
)
def test_none_result_passed_through(self, mock_conn, event_loop_fixture):
"""When result is None (e.g. first iteration), None is passed through."""
from collections import deque
tool_call_ids = {"web_search": deque(["tc-aaa"])}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, {})
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts, \
patch("acp_adapter.events.build_tool_complete") as mock_btc:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb(1, [{"name": "web_search", "result": None}])
mock_btc.assert_called_once_with("tc-aaa", "web_search", result=None, function_args=None, snapshot=None)
def test_step_callback_passes_arguments_and_snapshot(self, mock_conn, event_loop_fixture):
from collections import deque
tool_call_ids = {"write_file": deque(["tc-write"])}
tool_call_meta = {"tc-write": {"args": {"path": "fallback.txt"}, "snapshot": "snap"}}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts, \
patch("acp_adapter.events.build_tool_complete") as mock_btc:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb(1, [{"name": "write_file", "result": '{"bytes_written": 23}', "arguments": {"path": "diff-test.txt"}}])
mock_btc.assert_called_once_with(
"tc-write",
"write_file",
result='{"bytes_written": 23}',
function_args={"path": "diff-test.txt"},
snapshot="snap",
)
def test_tool_progress_captures_snapshot_metadata(self, mock_conn, event_loop_fixture):
tool_call_ids = {}
tool_call_meta = {}
loop = event_loop_fixture
with patch("acp_adapter.events.make_tool_call_id", return_value="tc-meta"), \
patch("acp_adapter.events._send_update") as mock_send, \
patch("agent.display.capture_local_edit_snapshot", return_value="snapshot"):
cb = make_tool_progress_cb(mock_conn, "session-1", loop, tool_call_ids, tool_call_meta)
cb("tool.started", "write_file", None, {"path": "diff-test.txt", "content": "hello"})
assert list(tool_call_ids["write_file"]) == ["tc-meta"]
assert tool_call_meta["tc-meta"] == {
"args": {"path": "diff-test.txt", "content": "hello"},
"snapshot": "snapshot",
}
mock_send.assert_called_once()
def test_todo_completion_emits_native_plan_update_after_tool_completion(self, mock_conn, event_loop_fixture):
from collections import deque
tool_call_ids = {"todo": deque(["tc-todo"])}
loop = event_loop_fixture
cb = make_step_cb(mock_conn, "session-1", loop, tool_call_ids, {})
todo_result = (
'{"todos":['
'{"id":"inspect","content":"Inspect ACP","status":"completed"},'
'{"id":"patch","content":"Patch renderer","status":"in_progress"},'
'{"id":"old","content":"Drop stale task","status":"cancelled"}'
'],"summary":{"total":3}}'
)
with patch("acp_adapter.events._send_update") as mock_send:
cb(1, [{"name": "todo", "result": todo_result}])
updates = [call.args[3] for call in mock_send.call_args_list]
assert [getattr(update, "session_update", None) for update in updates] == [
"tool_call_update",
"plan",
]
plan = updates[1]
assert isinstance(plan, AgentPlanUpdate)
assert [entry.content for entry in plan.entries] == [
"Inspect ACP",
"Patch renderer",
"[cancelled] Drop stale task",
]
assert [entry.status for entry in plan.entries] == ["completed", "in_progress", "completed"]
assert [entry.priority for entry in plan.entries] == ["medium", "medium", "medium"]
def test_todo_plan_update_parses_json_with_trailing_hint(self):
result = '{"todos":[{"id":"ship","content":"Ship ACP plan","status":"pending"}]}\n\n[Hint: persisted]'
update = _build_plan_update_from_todo_result(result)
assert isinstance(update, AgentPlanUpdate)
assert [entry.content for entry in update.entries] == ["Ship ACP plan"]
assert [entry.status for entry in update.entries] == ["pending"]
def test_todo_plan_update_with_empty_todos_clears_plan(self):
update = _build_plan_update_from_todo_result('{"todos":[],"summary":{"total":0}}')
assert isinstance(update, AgentPlanUpdate)
assert update.session_update == "plan"
assert update.entries == []
# ---------------------------------------------------------------------------
# Message callback
# ---------------------------------------------------------------------------
class TestMessageCallback:
def test_emits_agent_message_chunk(self, mock_conn, event_loop_fixture):
"""Message callback should emit AgentMessageChunk."""
loop = event_loop_fixture
cb = make_message_cb(mock_conn, "session-1", loop)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
future = MagicMock(spec=Future)
future.result.return_value = None
mock_rcts.return_value = future
cb("Here is your answer.")
mock_rcts.assert_called_once()
def test_ignores_empty_message(self, mock_conn, event_loop_fixture):
"""Empty text should not emit any update."""
loop = event_loop_fixture
cb = make_message_cb(mock_conn, "session-1", loop)
with patch("acp_adapter.events.asyncio.run_coroutine_threadsafe") as mock_rcts:
cb("")
mock_rcts.assert_not_called()
# ---------------------------------------------------------------------------
# Scheduler-failure regression
# ---------------------------------------------------------------------------
class TestSendUpdate:
def test_scheduler_failure_closes_update_coroutine(self, event_loop_fixture):
"""If run_coroutine_threadsafe raises, _send_update must close the coro."""
created = {"coro": None}
async def _session_update(session_id, update):
return None
conn = MagicMock()
def _capture_update(session_id, update):
created["coro"] = _session_update(session_id, update)
return created["coro"]
conn.session_update = _capture_update
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with patch(
"agent.async_utils.asyncio.run_coroutine_threadsafe",
side_effect=RuntimeError("scheduler down"),
):
_send_update(conn, "session-1", event_loop_fixture, {"type": "noop"})
gc.collect()
assert created["coro"] is not None
assert created["coro"].cr_frame is None
# Only count warnings about THIS test's coroutine; other tests in the
# same xdist worker (or stdlib mock internals) may emit unrelated
# "coroutine was never awaited" warnings that bleed through.
runtime_warnings = [
w for w in caught
if issubclass(w.category, RuntimeWarning)
and "was never awaited" in str(w.message)
and "_session_update" in str(w.message)
]
assert runtime_warnings == []
+362
View File
@@ -0,0 +1,362 @@
"""End-to-end tests for ACP MCP server registration and tool-result reporting.
Exercises the full flow through the ACP server layer:
new_session(mcpServers) → MCP tools registered → prompt() →
tool_progress_callback (ToolCallStart) →
step_callback with results (ToolCallUpdate with rawOutput) →
session_update events arrive at the mock client
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import acp
from acp.schema import (
EnvVariable,
HttpHeader,
McpServerHttp,
McpServerStdio,
NewSessionResponse,
PromptResponse,
TextContentBlock,
ToolCallProgress,
ToolCallStart,
)
from acp_adapter.server import HermesACPAgent
from acp_adapter.session import SessionManager
from acp_adapter.tools import build_tool_start
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture()
def mock_manager():
return SessionManager(agent_factory=lambda: MagicMock(name="MockAIAgent"))
@pytest.fixture()
def acp_agent(mock_manager):
return HermesACPAgent(session_manager=mock_manager)
# ---------------------------------------------------------------------------
# E2E: MCP registration → prompt → tool events
# ---------------------------------------------------------------------------
class TestMcpRegistrationE2E:
"""Full flow: session with MCP servers → prompt with tool calls → ACP events."""
@pytest.mark.asyncio
async def test_session_with_mcp_servers_registers_tools(self, acp_agent, mock_manager):
"""new_session with mcpServers converts them to Hermes config and registers."""
servers = [
McpServerStdio(
name="test-fs",
command="/usr/bin/mcp-fs",
args=["--root", "/tmp"],
env=[EnvVariable(name="DEBUG", value="1")],
),
McpServerHttp(
name="test-api",
url="https://api.example.com/mcp",
headers=[HttpHeader(name="Authorization", value="Bearer tok123")],
),
]
registered_configs = {}
def mock_register(config_map):
registered_configs.update(config_map)
return ["mcp_test_fs_read", "mcp_test_fs_write", "mcp_test_api_search"]
fake_tools = [
{"function": {"name": "mcp_test_fs_read"}},
{"function": {"name": "mcp_test_fs_write"}},
{"function": {"name": "mcp_test_api_search"}},
{"function": {"name": "terminal"}},
]
with patch("tools.mcp_tool.register_mcp_servers", side_effect=mock_register), \
patch("model_tools.get_tool_definitions", return_value=fake_tools):
resp = await acp_agent.new_session(cwd="/tmp", mcp_servers=servers)
assert isinstance(resp, NewSessionResponse)
state = mock_manager.get_session(resp.session_id)
# Verify stdio server was converted correctly
assert "test-fs" in registered_configs
fs_cfg = registered_configs["test-fs"]
assert fs_cfg["command"] == "/usr/bin/mcp-fs"
assert fs_cfg["args"] == ["--root", "/tmp"]
assert fs_cfg["env"] == {"DEBUG": "1"}
# Verify HTTP server was converted correctly
assert "test-api" in registered_configs
api_cfg = registered_configs["test-api"]
assert api_cfg["url"] == "https://api.example.com/mcp"
assert api_cfg["headers"] == {"Authorization": "Bearer tok123"}
# Verify agent tool surface was refreshed
assert state.agent.tools == fake_tools
assert state.agent.valid_tool_names == {
"mcp_test_fs_read", "mcp_test_fs_write", "mcp_test_api_search", "terminal"
}
@pytest.mark.asyncio
async def test_prompt_with_tool_calls_emits_acp_events(self, acp_agent, mock_manager):
"""Prompt → agent fires callbacks → ACP ToolCallStart + ToolCallUpdate events."""
resp = await acp_agent.new_session(cwd="/tmp")
session_id = resp.session_id
state = mock_manager.get_session(session_id)
# Wire up a mock ACP client connection
mock_conn = MagicMock(spec=acp.Client)
mock_conn.session_update = AsyncMock()
mock_conn.request_permission = AsyncMock()
acp_agent._conn = mock_conn
def mock_run_conversation(user_message, conversation_history=None, task_id=None, **kwargs):
"""Simulate an agent turn that calls terminal, gets a result, then responds."""
agent = state.agent
# 1) Agent fires tool_progress_callback (ToolCallStart)
if agent.tool_progress_callback:
agent.tool_progress_callback(
"tool.started", "terminal", "$ echo hello", {"command": "echo hello"}
)
# 2) Agent fires step_callback with tool results (ToolCallUpdate)
if agent.step_callback:
agent.step_callback(1, [
{"name": "terminal", "result": '{"output": "hello\\n", "exit_code": 0}'}
])
return {
"final_response": "The command output 'hello'.",
"messages": [
{"role": "user", "content": user_message},
{"role": "assistant", "content": "The command output 'hello'."},
],
}
state.agent.run_conversation = mock_run_conversation
prompt = [TextContentBlock(type="text", text="run echo hello")]
resp = await acp_agent.prompt(prompt=prompt, session_id=session_id)
assert isinstance(resp, PromptResponse)
assert resp.stop_reason == "end_turn"
# Collect all session_update calls
updates = []
for call in mock_conn.session_update.call_args_list:
# session_update(session_id, update) — grab the update
update_arg = call[1].get("update") or call[0][1]
updates.append(update_arg)
# Find tool_call (start) and tool_call_update (completion) events
starts = [u for u in updates if getattr(u, "session_update", None) == "tool_call"]
completions = [u for u in updates if getattr(u, "session_update", None) == "tool_call_update"]
# Should have at least one ToolCallStart for "terminal"
assert len(starts) >= 1, f"Expected ToolCallStart, got updates: {[getattr(u, 'session_update', '?') for u in updates]}"
start_event = starts[0]
assert isinstance(start_event, ToolCallStart)
assert start_event.title.startswith("terminal:")
# Should have at least one ToolCallUpdate (completion) with rawOutput
assert len(completions) >= 1, f"Expected ToolCallUpdate, got updates: {[getattr(u, 'session_update', '?') for u in updates]}"
complete_event = completions[0]
assert isinstance(complete_event, ToolCallProgress)
assert complete_event.status == "completed"
# Completion should contain human-readable output rather than forcing raw JSON panes.
assert complete_event.content
assert "hello" in complete_event.content[0].content.text
assert complete_event.raw_output is None
def test_patch_mode_tool_start_defers_diff_to_edit_approval_prompt(self):
update = build_tool_start(
"tc-1",
"patch",
{
"mode": "patch",
"patch": "*** Begin Patch\n*** Update File: src/app.py\n@@\n-old line\n+new line\n*** Add File: src/new.py\n+hello\n*** End Patch",
},
)
assert len(update.content) == 1
assert update.content[0].type == "content"
assert "Approval prompt shows the diff" in update.content[0].content.text
@pytest.mark.asyncio
async def test_prompt_tool_results_paired_by_call_id(self, acp_agent, mock_manager):
"""The ToolCallUpdate's toolCallId must match the ToolCallStart's."""
resp = await acp_agent.new_session(cwd="/tmp")
session_id = resp.session_id
state = mock_manager.get_session(session_id)
mock_conn = MagicMock(spec=acp.Client)
mock_conn.session_update = AsyncMock()
mock_conn.request_permission = AsyncMock()
acp_agent._conn = mock_conn
def mock_run(user_message, conversation_history=None, task_id=None, **kwargs):
agent = state.agent
# Fire two tool calls
if agent.tool_progress_callback:
agent.tool_progress_callback("tool.started", "read_file", "read: /etc/hosts", {"path": "/etc/hosts"})
agent.tool_progress_callback("tool.started", "web_search", "web search: test", {"query": "test"})
if agent.step_callback:
agent.step_callback(1, [
{"name": "read_file", "result": '{"content": "127.0.0.1 localhost"}'},
{"name": "web_search", "result": '{"data": {"web": []}}'},
])
return {"final_response": "Done.", "messages": []}
state.agent.run_conversation = mock_run
prompt = [TextContentBlock(type="text", text="test")]
await acp_agent.prompt(prompt=prompt, session_id=session_id)
updates = []
for call in mock_conn.session_update.call_args_list:
update_arg = call[1].get("update") or call[0][1]
updates.append(update_arg)
starts = [u for u in updates if getattr(u, "session_update", None) == "tool_call"]
completions = [u for u in updates if getattr(u, "session_update", None) == "tool_call_update"]
assert len(starts) == 2, f"Expected 2 starts, got {len(starts)}"
assert len(completions) == 2, f"Expected 2 completions, got {len(completions)}"
# Each completion's toolCallId must match a start's toolCallId
start_ids = {s.tool_call_id for s in starts}
completion_ids = {c.tool_call_id for c in completions}
assert start_ids == completion_ids, (
f"IDs must match: starts={start_ids}, completions={completion_ids}"
)
class TestMcpSanitizationE2E:
"""Verify server names with special chars work end-to-end."""
@pytest.mark.asyncio
async def test_slashed_server_name_registers_cleanly(self, acp_agent, mock_manager):
"""Server name 'ai.exa/exa' should not crash — tools get sanitized names."""
servers = [
McpServerHttp(
name="ai.exa/exa",
url="https://exa.ai/mcp",
headers=[],
),
]
registered_configs = {}
def mock_register(config_map):
registered_configs.update(config_map)
return ["mcp_ai_exa_exa_search"]
fake_tools = [{"function": {"name": "mcp_ai_exa_exa_search"}}]
with patch("tools.mcp_tool.register_mcp_servers", side_effect=mock_register), \
patch("model_tools.get_tool_definitions", return_value=fake_tools):
resp = await acp_agent.new_session(cwd="/tmp", mcp_servers=servers)
state = mock_manager.get_session(resp.session_id)
# Raw server name preserved as config key
assert "ai.exa/exa" in registered_configs
# Agent tools refreshed with sanitized name
assert "mcp_ai_exa_exa_search" in state.agent.valid_tool_names
class TestSessionLifecycleMcpE2E:
"""Verify MCP servers are registered on all session lifecycle methods."""
@pytest.mark.asyncio
async def test_load_session_registers_mcp(self, acp_agent, mock_manager):
"""load_session re-registers MCP servers (spec says agents may not retain them)."""
# Create a session first
create_resp = await acp_agent.new_session(cwd="/tmp")
sid = create_resp.session_id
servers = [
McpServerStdio(name="srv", command="/bin/test", args=[], env=[]),
]
registered = {}
def mock_register(config_map):
registered.update(config_map)
return []
state = mock_manager.get_session(sid)
state.agent.enabled_toolsets = ["hermes-acp"]
state.agent.disabled_toolsets = None
state.agent.tools = []
state.agent.valid_tool_names = set()
with patch("tools.mcp_tool.register_mcp_servers", side_effect=mock_register), \
patch("model_tools.get_tool_definitions", return_value=[]):
await acp_agent.load_session(cwd="/tmp", session_id=sid, mcp_servers=servers)
assert "srv" in registered
@pytest.mark.asyncio
async def test_resume_session_registers_mcp(self, acp_agent, mock_manager):
"""resume_session re-registers MCP servers."""
create_resp = await acp_agent.new_session(cwd="/tmp")
sid = create_resp.session_id
servers = [
McpServerStdio(name="srv2", command="/bin/test2", args=[], env=[]),
]
registered = {}
def mock_register(config_map):
registered.update(config_map)
return []
state = mock_manager.get_session(sid)
state.agent.enabled_toolsets = ["hermes-acp"]
state.agent.disabled_toolsets = None
state.agent.tools = []
state.agent.valid_tool_names = set()
with patch("tools.mcp_tool.register_mcp_servers", side_effect=mock_register), \
patch("model_tools.get_tool_definitions", return_value=[]):
await acp_agent.resume_session(cwd="/tmp", session_id=sid, mcp_servers=servers)
assert "srv2" in registered
@pytest.mark.asyncio
async def test_fork_session_registers_mcp(self, acp_agent, mock_manager):
"""fork_session registers MCP servers on the new forked session."""
create_resp = await acp_agent.new_session(cwd="/tmp")
sid = create_resp.session_id
servers = [
McpServerHttp(name="api", url="https://api.test/mcp", headers=[]),
]
registered = {}
def mock_register(config_map):
registered.update(config_map)
return []
# Need to set up the forked session's agent too
with patch("tools.mcp_tool.register_mcp_servers", side_effect=mock_register), \
patch("model_tools.get_tool_definitions", return_value=[]):
fork_resp = await acp_agent.fork_session(
cwd="/tmp", session_id=sid, mcp_servers=servers
)
assert fork_resp.session_id != ""
assert "api" in registered
+233
View File
@@ -0,0 +1,233 @@
"""Tests for acp_adapter.permissions."""
import asyncio
import inspect
from concurrent.futures import Future
from unittest.mock import AsyncMock, MagicMock, patch
from acp.schema import (
AllowedOutcome,
DeniedOutcome,
RequestPermissionResponse,
)
from acp_adapter.permissions import make_approval_callback
from tools.approval import prompt_dangerous_approval
def _make_response(outcome):
return RequestPermissionResponse(outcome=outcome)
def _invoke_callback(
outcome,
*,
allow_permanent=True,
timeout=60.0,
use_prompt_path=False,
):
loop = MagicMock(spec=asyncio.AbstractEventLoop)
request_permission = AsyncMock(name="request_permission")
future = MagicMock(spec=Future)
future.result.return_value = _make_response(outcome)
scheduled = {}
def _schedule(coro, passed_loop):
scheduled["coro"] = coro
scheduled["loop"] = passed_loop
return future
with patch("agent.async_utils.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=timeout)
if use_prompt_path:
result = prompt_dangerous_approval(
"rm -rf /",
"dangerous command",
allow_permanent=allow_permanent,
approval_callback=cb,
)
else:
result = cb(
"rm -rf /",
"dangerous command",
allow_permanent=allow_permanent,
)
scheduled["coro"].close()
_, kwargs = request_permission.call_args
return result, kwargs, scheduled, future, loop
class TestApprovalBridge:
def test_bridge_schedules_request_on_the_given_loop(self):
result, kwargs, scheduled, _, loop = _invoke_callback(
AllowedOutcome(option_id="allow_once", outcome="selected"),
)
tool_call = kwargs["tool_call"]
option_ids = [option.option_id for option in kwargs["options"]]
assert result == "once"
assert scheduled["loop"] is loop
assert inspect.iscoroutine(scheduled["coro"])
assert kwargs["session_id"] == "s1"
assert tool_call.session_update == "tool_call_update"
assert tool_call.tool_call_id.startswith("perm-check-")
assert tool_call.kind == "execute"
assert tool_call.status == "pending"
assert "dangerous command" in tool_call.title
assert "rm -rf /" in tool_call.title
content_text = tool_call.content[0].content.text
assert "$ rm -rf /" in content_text
assert "dangerous command" in content_text
assert tool_call.raw_input == {
"command": "rm -rf /",
"description": "dangerous command",
}
assert option_ids == [
"allow_once",
"allow_session",
"allow_always",
"deny",
"deny_always",
]
def test_tool_call_ids_are_unique(self):
_, first_kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_once", outcome="selected"),
)
_, second_kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_once", outcome="selected"),
)
assert first_kwargs["tool_call"].tool_call_id != second_kwargs["tool_call"].tool_call_id
def test_prompt_path_keeps_session_option_when_permanent_disabled(self):
result, kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_session", outcome="selected"),
allow_permanent=False,
use_prompt_path=True,
)
option_ids = [option.option_id for option in kwargs["options"]]
assert result == "session"
assert option_ids == ["allow_once", "allow_session", "deny", "deny_always"]
def test_reject_always_outcome_denies_without_changing_policy(self):
result, kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="deny_always", outcome="selected"),
use_prompt_path=True,
)
deny_always = [option for option in kwargs["options"] if option.option_id == "deny_always"]
assert result == "deny"
assert len(deny_always) == 1
assert deny_always[0].kind == "reject_always"
def test_allow_always_maps_correctly(self):
result, _, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_always", outcome="selected"),
use_prompt_path=True,
)
assert result == "always"
def test_denied_and_unknown_outcomes_deny(self):
denied_result, _, _, _, _ = _invoke_callback(DeniedOutcome(outcome="cancelled"))
unknown_result, _, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="unexpected", outcome="selected"),
)
assert denied_result == "deny"
assert unknown_result == "deny"
def test_timeout_returns_deny_and_cancels_future(self):
loop = MagicMock(spec=asyncio.AbstractEventLoop)
request_permission = AsyncMock(name="request_permission")
future = MagicMock(spec=Future)
future.result.side_effect = TimeoutError("timed out")
scheduled = {}
def _schedule(coro, passed_loop):
scheduled["coro"] = coro
scheduled["loop"] = passed_loop
return future
with patch("agent.async_utils.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=0.01)
result = cb("rm -rf /", "dangerous command")
scheduled["coro"].close()
assert result == "deny"
assert scheduled["loop"] is loop
assert future.cancel.call_count == 1
def test_none_response_returns_deny(self):
"""When request_permission resolves to None, the callback returns 'deny'."""
loop = MagicMock(spec=asyncio.AbstractEventLoop)
request_permission = AsyncMock(name="request_permission")
future = MagicMock(spec=Future)
future.result.return_value = None
scheduled = {}
def _schedule(coro, passed_loop):
scheduled["coro"] = coro
scheduled["loop"] = passed_loop
return future
with patch("agent.async_utils.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=1.0)
result = cb("echo hi", "demo")
scheduled["coro"].close()
assert result == "deny"
# ---------------------------------------------------------------------------
# Scheduler-failure regression
# ---------------------------------------------------------------------------
import gc # noqa: E402
import warnings # noqa: E402
class TestSchedulerFailure:
def test_scheduler_failure_closes_permission_coroutine(self):
"""If run_coroutine_threadsafe raises, the coro is closed and we return 'deny'."""
loop = MagicMock(spec=asyncio.AbstractEventLoop)
created = {"coro": None}
async def _response_coro(**kwargs):
return _make_response(AllowedOutcome(option_id="allow_once", outcome="selected"))
def _request_permission(**kwargs):
created["coro"] = _response_coro(**kwargs)
return created["coro"]
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with patch(
"agent.async_utils.asyncio.run_coroutine_threadsafe",
side_effect=RuntimeError("scheduler down"),
):
cb = make_approval_callback(_request_permission, loop, session_id="s1", timeout=0.01)
result = cb("rm -rf /", "dangerous")
gc.collect()
assert result == "deny"
assert created["coro"] is not None
assert created["coro"].cr_frame is None
runtime_warnings = [
w for w in caught
if issubclass(w.category, RuntimeWarning)
and "was never awaited" in str(w.message)
and "_response_coro" in str(w.message)
]
assert runtime_warnings == []
+210
View File
@@ -0,0 +1,210 @@
"""Tests for acp_adapter.entry._BenignProbeMethodFilter.
Covers both the isolated filter logic and the full end-to-end path where a
client sends a bare JSON-RPC ``ping`` request over stdio and the acp runtime
surfaces the resulting ``RequestError`` via ``logging.exception("Background
task failed", ...)``.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
from io import StringIO
import pytest
from acp.exceptions import RequestError
from acp_adapter.entry import _BenignProbeMethodFilter
# -- Unit tests on the filter itself ----------------------------------------
def _make_record(msg: str, exc: BaseException | None) -> logging.LogRecord:
record = logging.LogRecord(
name="root",
level=logging.ERROR,
pathname=__file__,
lineno=0,
msg=msg,
args=(),
exc_info=(type(exc), exc, exc.__traceback__) if exc else None,
)
return record
def _bake_tb(exc: BaseException) -> BaseException:
try:
raise exc
except BaseException as e: # noqa: BLE001
return e
@pytest.mark.parametrize("method", ["ping", "health", "healthcheck"])
def test_filter_suppresses_benign_probe(method: str) -> None:
f = _BenignProbeMethodFilter()
exc = _bake_tb(RequestError.method_not_found(method))
record = _make_record("Background task failed", exc)
assert f.filter(record) is False
def test_filter_allows_real_method_not_found() -> None:
f = _BenignProbeMethodFilter()
exc = _bake_tb(RequestError.method_not_found("session/custom"))
record = _make_record("Background task failed", exc)
assert f.filter(record) is True
def test_filter_allows_non_request_error() -> None:
f = _BenignProbeMethodFilter()
exc = _bake_tb(RuntimeError("boom"))
record = _make_record("Background task failed", exc)
assert f.filter(record) is True
def test_filter_allows_different_message_even_for_ping() -> None:
"""Only 'Background task failed' is muted — other messages pass through."""
f = _BenignProbeMethodFilter()
exc = _bake_tb(RequestError.method_not_found("ping"))
record = _make_record("Some other context", exc)
assert f.filter(record) is True
def test_filter_allows_request_error_with_different_code() -> None:
f = _BenignProbeMethodFilter()
exc = _bake_tb(RequestError.invalid_params({"method": "ping"}))
record = _make_record("Background task failed", exc)
assert f.filter(record) is True
def test_filter_allows_log_without_exc_info() -> None:
f = _BenignProbeMethodFilter()
record = _make_record("Background task failed", None)
assert f.filter(record) is True
# -- End-to-end: drive a real JSON-RPC `ping` through acp.run_agent ---------
class _FakeAgent:
"""Minimal acp.Agent stub — we only need the router to build."""
async def initialize(self, **kwargs): # noqa: ANN003
from acp.schema import AgentCapabilities, InitializeResponse
return InitializeResponse(protocol_version=1, agent_capabilities=AgentCapabilities())
async def new_session(self, cwd, mcp_servers=None, **kwargs): # noqa: ANN001, ANN003
from acp.schema import NewSessionResponse
return NewSessionResponse(session_id="test")
async def prompt(self, session_id, prompt, **kwargs): # noqa: ANN001, ANN003
from acp.schema import PromptResponse
return PromptResponse(stop_reason="end_turn")
async def cancel(self, session_id, **kwargs): # noqa: ANN001, ANN003
pass
async def authenticate(self, **kwargs): # noqa: ANN003
pass
def on_connect(self, conn): # noqa: ANN001
pass
@pytest.mark.asyncio
async def test_bare_ping_request_produces_proper_response_and_no_stderr_noise(
caplog: pytest.LogCaptureFixture,
) -> None:
"""A bare ``ping`` must get a JSON-RPC -32601 back AND leave stderr clean
when the filter is installed on the handler.
"""
import acp
# Attach the filter to a fresh stream handler that mirrors entry._setup_logging.
stream = StringIO()
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter("%(name)s|%(levelname)s|%(message)s"))
handler.addFilter(_BenignProbeMethodFilter())
root = logging.getLogger()
prior_handlers = root.handlers[:]
prior_level = root.level
root.handlers = [handler]
root.setLevel(logging.INFO)
# Also suppress propagation of caplog's default handler interfering with
# our stream (caplog still captures via its own propagation hook).
try:
loop = asyncio.get_running_loop()
# Pipe client -> agent
client_to_agent_r, client_to_agent_w = os.pipe()
# Pipe agent -> client
agent_to_client_r, agent_to_client_w = os.pipe()
in_read_file = os.fdopen(client_to_agent_r, "rb", buffering=0)
in_write_file = os.fdopen(client_to_agent_w, "wb", buffering=0)
out_read_file = os.fdopen(agent_to_client_r, "rb", buffering=0)
out_write_file = os.fdopen(agent_to_client_w, "wb", buffering=0)
# Agent reads its input from this StreamReader:
agent_input = asyncio.StreamReader(limit=1024 * 1024, loop=loop)
agent_input_proto = asyncio.StreamReaderProtocol(agent_input, loop=loop)
await loop.connect_read_pipe(lambda: agent_input_proto, in_read_file)
# Agent writes its output via this StreamWriter:
out_transport, out_protocol = await loop.connect_write_pipe(
asyncio.streams.FlowControlMixin, out_write_file
)
agent_output = asyncio.StreamWriter(out_transport, out_protocol, None, loop)
# Test harness reads agent output via this StreamReader:
client_input = asyncio.StreamReader(limit=1024 * 1024, loop=loop)
client_input_proto = asyncio.StreamReaderProtocol(client_input, loop=loop)
await loop.connect_read_pipe(lambda: client_input_proto, out_read_file)
agent_task = asyncio.create_task(
acp.run_agent(
_FakeAgent(),
input_stream=agent_output,
output_stream=agent_input,
use_unstable_protocol=True,
)
)
# Send a bare `ping`
request = {"jsonrpc": "2.0", "id": 1, "method": "ping", "params": {}}
in_write_file.write((json.dumps(request) + "\n").encode())
in_write_file.flush()
response_line = await asyncio.wait_for(client_input.readline(), timeout=5.0)
# Give the supervisor task a tick to fire (filter should eat it)
await asyncio.sleep(0.2)
response = json.loads(response_line.decode())
assert response["error"]["code"] == -32601, response
assert response["error"]["data"] == {"method": "ping"}, response
logs = stream.getvalue()
assert "Background task failed" not in logs, (
f"ping noise leaked to stderr:\n{logs}"
)
# Clean shutdown
in_write_file.close()
try:
await asyncio.wait_for(agent_task, timeout=2.0)
except (asyncio.TimeoutError, Exception):
agent_task.cancel()
try:
await agent_task
except BaseException: # noqa: BLE001
pass
finally:
root.handlers = prior_handlers
root.setLevel(prior_level)
+90
View File
@@ -0,0 +1,90 @@
"""Tests for ACP Registry metadata shipped with Hermes."""
from __future__ import annotations
import json
import re
import tomllib
from pathlib import Path
import xml.etree.ElementTree as ET
ROOT = Path(__file__).resolve().parents[2]
MANIFEST = ROOT / "acp_registry" / "agent.json"
ICON = ROOT / "acp_registry" / "icon.svg"
FORBIDDEN_MANIFEST_KEYS = {"schema_version", "display_name"}
ALLOWED_DISTRIBUTIONS = {"binary", "npx", "uvx"}
def _manifest() -> dict:
return json.loads(MANIFEST.read_text(encoding="utf-8"))
def _pyproject_version() -> str:
data = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
return data["project"]["version"]
def test_agent_json_matches_official_registry_required_fields():
data = _manifest()
assert FORBIDDEN_MANIFEST_KEYS.isdisjoint(data)
assert data["id"] == "hermes-agent"
assert re.fullmatch(r"[a-z][a-z0-9-]*", data["id"])
assert data["name"] == "Hermes Agent"
assert data["description"]
assert data["repository"] == "https://github.com/NousResearch/hermes-agent"
assert data["website"].startswith("https://hermes-agent.nousresearch.com/")
assert data["authors"] == ["Nous Research"]
assert data["license"] == "MIT"
assert set(data["distribution"]) <= ALLOWED_DISTRIBUTIONS
def test_agent_json_uses_uvx_distribution_without_local_command_fields():
data = _manifest()
assert set(data["distribution"]) == {"uvx"}
uvx = data["distribution"]["uvx"]
# Schema allows {package, args, env}; we use {package, args}.
assert set(uvx) <= {"package", "args", "env"}
assert "package" in uvx
assert uvx["package"] == f"hermes-agent[acp]=={data['version']}"
assert uvx["args"] == ["hermes-acp"]
# Old command-shape fields must not leak back in.
assert "type" not in data["distribution"]
assert "command" not in data["distribution"]
def test_agent_json_version_matches_pyproject():
assert _manifest()["version"] == _pyproject_version()
def test_agent_json_pins_uvx_package_to_pyproject_version():
"""The registry CI rejects ``@latest`` and floating pins; the manifest must
always reference the exact PyPI version listed in pyproject.toml."""
assert _manifest()["distribution"]["uvx"]["package"] == (
f"hermes-agent[acp]=={_pyproject_version()}"
)
def test_icon_svg_is_16x16_current_color():
root = ET.fromstring(ICON.read_text(encoding="utf-8"))
assert root.attrib["viewBox"] == "0 0 16 16"
assert root.attrib["width"] == "16"
assert root.attrib["height"] == "16"
def test_icon_svg_has_no_hardcoded_colors_or_gradients():
text = ICON.read_text(encoding="utf-8")
assert "linearGradient" not in text
assert "radialGradient" not in text
assert "url(#" not in text
assert not re.search(r"#[0-9a-fA-F]{3,8}\b", text)
root = ET.fromstring(text)
for element in root.iter():
for attr in ("fill", "stroke"):
value = element.attrib.get(attr)
if value is not None:
assert value in {"currentColor", "none"}
File diff suppressed because it is too large Load Diff
+598
View File
@@ -0,0 +1,598 @@
"""Tests for acp_adapter.session — SessionManager and SessionState."""
import contextlib
import io
import json
import time
from types import SimpleNamespace
import pytest
from unittest.mock import MagicMock, patch
from acp_adapter import session as acp_session
from acp_adapter.session import SessionManager, SessionState
from hermes_state import SessionDB
def _mock_agent():
return MagicMock(name="MockAIAgent")
@pytest.fixture()
def manager():
"""SessionManager with a mock agent factory (avoids needing API keys)."""
return SessionManager(agent_factory=_mock_agent)
# ---------------------------------------------------------------------------
# create / get
# ---------------------------------------------------------------------------
class TestCreateSession:
def test_create_session_returns_state(self, manager):
state = manager.create_session(cwd="/tmp/work")
assert isinstance(state, SessionState)
assert state.cwd == "/tmp/work"
assert state.session_id
assert state.history == []
assert state.agent is not None
def test_create_session_registers_task_cwd(self, manager, monkeypatch):
calls = []
monkeypatch.setattr("acp_adapter.session._register_task_cwd", lambda task_id, cwd: calls.append((task_id, cwd)))
state = manager.create_session(cwd="/tmp/work")
assert calls == [(state.session_id, "/tmp/work")]
def test_register_task_cwd_translates_windows_drive_for_wsl_tools(self, monkeypatch):
captured = {}
def fake_register_task_env_overrides(task_id, overrides):
captured["task_id"] = task_id
captured["overrides"] = overrides
monkeypatch.setattr("hermes_constants._wsl_detected", True)
monkeypatch.setattr(
"tools.terminal_tool.register_task_env_overrides",
fake_register_task_env_overrides,
)
acp_session._register_task_cwd("session-1", r"E:\Projects\AI\paperclip")
assert captured == {
"task_id": "session-1",
"overrides": {"cwd": "/mnt/e/Projects/AI/paperclip"},
}
def test_session_ids_are_unique(self, manager):
s1 = manager.create_session()
s2 = manager.create_session()
assert s1.session_id != s2.session_id
def test_get_session(self, manager):
state = manager.create_session()
fetched = manager.get_session(state.session_id)
assert fetched is state
def test_get_nonexistent_session_returns_none(self, manager):
assert manager.get_session("does-not-exist") is None
# ---------------------------------------------------------------------------
# WSL cwd translation
# ---------------------------------------------------------------------------
class TestWslCwdTranslation:
def test_translate_acp_cwd_converts_windows_drive_path_when_wsl(self, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", True)
assert acp_session._translate_acp_cwd(r"E:\Projects\AI\paperclip") == "/mnt/e/Projects/AI/paperclip"
def test_translate_acp_cwd_handles_forward_slashes_when_wsl(self, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", True)
assert acp_session._translate_acp_cwd("D:/work/project") == "/mnt/d/work/project"
def test_translate_acp_cwd_leaves_windows_drive_path_unchanged_off_wsl(self, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", False)
assert acp_session._translate_acp_cwd(r"E:\Projects\AI\paperclip") == r"E:\Projects\AI\paperclip"
def test_translate_acp_cwd_leaves_posix_path_unchanged_on_wsl(self, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", True)
assert acp_session._translate_acp_cwd("/mnt/e/Projects/AI/paperclip") == "/mnt/e/Projects/AI/paperclip"
def test_create_session_stores_translated_cwd_on_wsl(self, manager, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", True)
state = manager.create_session(cwd=r"E:\Projects\AI\paperclip")
assert state.cwd == "/mnt/e/Projects/AI/paperclip"
def test_fork_session_stores_translated_cwd_on_wsl(self, manager, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", True)
original = manager.create_session(cwd="/tmp/base")
forked = manager.fork_session(original.session_id, cwd=r"D:\work\project")
assert forked is not None
assert forked.cwd == "/mnt/d/work/project"
def test_update_cwd_stores_translated_cwd_on_wsl(self, manager, monkeypatch):
monkeypatch.setattr("hermes_constants._wsl_detected", True)
state = manager.create_session(cwd="/tmp/old")
updated = manager.update_cwd(state.session_id, cwd=r"C:\Users\foo\project")
assert updated is not None
assert updated.cwd == "/mnt/c/Users/foo/project"
# ---------------------------------------------------------------------------
# fork
# ---------------------------------------------------------------------------
class TestForkSession:
def test_fork_session_deep_copies_history(self, manager):
original = manager.create_session()
original.history.append({"role": "user", "content": "hello"})
original.history.append({"role": "assistant", "content": "hi"})
forked = manager.fork_session(original.session_id, cwd="/new")
assert forked is not None
# History should be equal in content
assert len(forked.history) == 2
assert forked.history[0]["content"] == "hello"
# But a deep copy — mutating one doesn't affect the other
forked.history.append({"role": "user", "content": "extra"})
assert len(original.history) == 2
assert len(forked.history) == 3
def test_fork_session_has_new_id(self, manager):
original = manager.create_session()
forked = manager.fork_session(original.session_id)
assert forked is not None
assert forked.session_id != original.session_id
def test_fork_nonexistent_returns_none(self, manager):
assert manager.fork_session("bogus-id") is None
# ---------------------------------------------------------------------------
# list / cleanup / remove
# ---------------------------------------------------------------------------
class TestListAndCleanup:
def test_list_sessions_empty(self, manager):
assert manager.list_sessions() == []
def test_list_sessions_returns_created(self, manager):
s1 = manager.create_session(cwd="/a")
s2 = manager.create_session(cwd="/b")
s1.history.append({"role": "user", "content": "hello from a"})
s2.history.append({"role": "user", "content": "hello from b"})
listing = manager.list_sessions()
ids = {s["session_id"] for s in listing}
assert s1.session_id in ids
assert s2.session_id in ids
assert len(listing) == 2
def test_list_sessions_hides_empty_threads(self, manager):
manager.create_session(cwd="/empty")
assert manager.list_sessions() == []
def test_save_session_preserves_existing_messages_on_encode_failure(self, manager):
"""Regression for #13675: a bad message in state.history must not
clobber the previously-persisted transcript. replace_messages()
wraps DELETE + INSERT in a single rolled-back-on-exception txn.
"""
state = manager.create_session()
state.history.append({"role": "user", "content": "original"})
manager.save_session(state.session_id)
# Now swap history with a message whose tool_calls is non-JSON-serializable.
# _execute_write rolls back; the previously persisted "original" stays.
state.history = [
{"role": "user", "content": "replacement"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"bad": object()}],
},
]
manager.save_session(state.session_id)
db = manager._get_db()
messages = db.get_messages_as_conversation(state.session_id)
assert messages == [{"role": "user", "content": "original"}]
def test_cleanup_clears_all(self, manager):
s1 = manager.create_session()
s2 = manager.create_session()
s1.history.append({"role": "user", "content": "one"})
s2.history.append({"role": "user", "content": "two"})
assert len(manager.list_sessions()) == 2
manager.cleanup()
assert manager.list_sessions() == []
def test_remove_session(self, manager):
state = manager.create_session()
assert manager.remove_session(state.session_id) is True
assert manager.get_session(state.session_id) is None
# Removing again returns False
assert manager.remove_session(state.session_id) is False
# ---------------------------------------------------------------------------
# persistence — sessions survive process restarts (via SessionDB)
# ---------------------------------------------------------------------------
class TestPersistence:
"""Verify that sessions are persisted to SessionDB and can be restored."""
def test_create_session_includes_registered_mcp_toolsets(self, tmp_path, monkeypatch):
captured = {}
def fake_resolve_runtime_provider(requested=None, **kwargs):
return {
"provider": "openrouter",
"api_mode": "chat_completions",
"base_url": "https://openrouter.example/v1",
"api_key": "***",
"command": None,
"args": [],
}
def fake_agent(**kwargs):
captured.update(kwargs)
return SimpleNamespace(model=kwargs.get("model"), enabled_toolsets=kwargs.get("enabled_toolsets"))
monkeypatch.setattr("hermes_cli.config.load_config", lambda: {
"model": {"provider": "openrouter", "default": "test-model"},
"mcp_servers": {
"olympus": {"command": "python", "enabled": True},
"exa": {"url": "https://exa.ai/mcp"},
"disabled": {"command": "python", "enabled": False},
},
})
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
fake_resolve_runtime_provider,
)
db = SessionDB(tmp_path / "state.db")
with patch("run_agent.AIAgent", side_effect=fake_agent):
manager = SessionManager(db=db)
manager.create_session(cwd="/work")
assert captured["enabled_toolsets"] == ["hermes-acp", "mcp-olympus", "mcp-exa"]
def test_create_session_writes_to_db(self, manager):
state = manager.create_session(cwd="/project")
db = manager._get_db()
assert db is not None
row = db.get_session(state.session_id)
assert row is not None
assert row["source"] == "acp"
# cwd stored in model_config JSON
mc = json.loads(row["model_config"])
assert mc["cwd"] == "/project"
def test_get_session_restores_from_db(self, manager):
"""Simulate process restart: create session, drop from memory, get again."""
state = manager.create_session(cwd="/work")
state.history.append({"role": "user", "content": "hello"})
state.history.append({"role": "assistant", "content": "hi there"})
manager.save_session(state.session_id)
sid = state.session_id
# Drop from in-memory store (simulates process restart).
with manager._lock:
del manager._sessions[sid]
# get_session should transparently restore from DB.
restored = manager.get_session(sid)
assert restored is not None
assert restored.session_id == sid
assert restored.cwd == "/work"
assert len(restored.history) == 2
assert restored.history[0]["content"] == "hello"
assert restored.history[1]["content"] == "hi there"
# Agent should have been recreated.
assert restored.agent is not None
def test_save_session_updates_db(self, manager):
state = manager.create_session()
state.history.append({"role": "user", "content": "test"})
manager.save_session(state.session_id)
db = manager._get_db()
messages = db.get_messages_as_conversation(state.session_id)
assert len(messages) == 1
assert messages[0]["content"] == "test"
def test_remove_session_deletes_from_db(self, manager):
state = manager.create_session()
db = manager._get_db()
assert db.get_session(state.session_id) is not None
manager.remove_session(state.session_id)
assert db.get_session(state.session_id) is None
def test_cleanup_removes_all_from_db(self, manager):
s1 = manager.create_session()
s2 = manager.create_session()
db = manager._get_db()
assert db.get_session(s1.session_id) is not None
assert db.get_session(s2.session_id) is not None
manager.cleanup()
assert db.get_session(s1.session_id) is None
assert db.get_session(s2.session_id) is None
def test_list_sessions_includes_db_only(self, manager):
"""Sessions only in DB (not in memory) appear in list_sessions."""
state = manager.create_session(cwd="/db-only")
state.history.append({"role": "user", "content": "database only thread"})
manager.save_session(state.session_id)
sid = state.session_id
# Drop from memory.
with manager._lock:
del manager._sessions[sid]
listing = manager.list_sessions()
ids = {s["session_id"] for s in listing}
assert sid in ids
def test_list_sessions_filters_by_cwd(self, manager):
keep = manager.create_session(cwd="/keep")
drop = manager.create_session(cwd="/drop")
keep.history.append({"role": "user", "content": "keep me"})
drop.history.append({"role": "user", "content": "drop me"})
listing = manager.list_sessions(cwd="/keep")
ids = {s["session_id"] for s in listing}
assert keep.session_id in ids
assert drop.session_id not in ids
def test_list_sessions_matches_windows_and_wsl_paths(self, manager):
state = manager.create_session(cwd="/mnt/e/Projects/AI/browser-link-3")
state.history.append({"role": "user", "content": "same project from WSL"})
listing = manager.list_sessions(cwd=r"E:\Projects\AI\browser-link-3")
ids = {s["session_id"] for s in listing}
assert state.session_id in ids
def test_list_sessions_prefers_title_then_preview(self, manager):
state = manager.create_session(cwd="/named")
state.history.append({"role": "user", "content": "Investigate broken ACP history in Zed"})
manager.save_session(state.session_id)
db = manager._get_db()
db.set_session_title(state.session_id, "Fix Zed ACP history")
listing = manager.list_sessions(cwd="/named")
assert listing[0]["title"] == "Fix Zed ACP history"
db.set_session_title(state.session_id, "")
listing = manager.list_sessions(cwd="/named")
assert listing[0]["title"].startswith("Investigate broken ACP history")
def test_list_sessions_sorted_by_most_recent_activity(self, manager):
older = manager.create_session(cwd="/ordered")
older.history.append({"role": "user", "content": "older"})
manager.save_session(older.session_id)
time.sleep(0.02)
newer = manager.create_session(cwd="/ordered")
newer.history.append({"role": "user", "content": "newer"})
manager.save_session(newer.session_id)
listing = manager.list_sessions(cwd="/ordered")
assert [item["session_id"] for item in listing[:2]] == [newer.session_id, older.session_id]
assert listing[0]["updated_at"]
assert listing[1]["updated_at"]
def test_fork_restores_source_from_db(self, manager):
"""Forking a session that is only in DB should work."""
original = manager.create_session()
original.history.append({"role": "user", "content": "context"})
manager.save_session(original.session_id)
# Drop original from memory.
with manager._lock:
del manager._sessions[original.session_id]
forked = manager.fork_session(original.session_id, cwd="/fork")
assert forked is not None
assert len(forked.history) == 1
assert forked.history[0]["content"] == "context"
assert forked.session_id != original.session_id
def test_update_cwd_restores_from_db(self, manager):
state = manager.create_session(cwd="/old")
sid = state.session_id
with manager._lock:
del manager._sessions[sid]
updated = manager.update_cwd(sid, "/new")
assert updated is not None
assert updated.cwd == "/new"
# Should also be persisted in DB.
db = manager._get_db()
row = db.get_session(sid)
mc = json.loads(row["model_config"])
assert mc["cwd"] == "/new"
def test_only_restores_acp_sessions(self, manager):
"""get_session should not restore non-ACP sessions from DB."""
db = manager._get_db()
# Manually create a CLI session in the DB.
db.create_session(session_id="cli-session-123", source="cli", model="test")
# Should not be found via ACP SessionManager.
assert manager.get_session("cli-session-123") is None
def test_sessions_searchable_via_fts(self, manager):
"""ACP sessions stored in SessionDB are searchable via FTS5."""
state = manager.create_session()
state.history.append({"role": "user", "content": "how do I configure nginx"})
state.history.append({"role": "assistant", "content": "Here is the nginx config..."})
manager.save_session(state.session_id)
db = manager._get_db()
results = db.search_messages("nginx")
assert len(results) > 0
session_ids = {r["session_id"] for r in results}
assert state.session_id in session_ids
def test_tool_calls_persisted(self, manager):
"""Messages with tool_calls should round-trip through the DB."""
state = manager.create_session()
state.history.append({
"role": "assistant",
"content": None,
"tool_calls": [{"id": "tc_1", "type": "function",
"function": {"name": "terminal", "arguments": "{}"}}],
})
state.history.append({
"role": "tool",
"content": "output here",
"tool_call_id": "tc_1",
"name": "terminal",
})
manager.save_session(state.session_id)
# Drop from memory, restore from DB.
with manager._lock:
del manager._sessions[state.session_id]
restored = manager.get_session(state.session_id)
assert restored is not None
assert len(restored.history) == 2
assert restored.history[0].get("tool_calls") is not None
assert restored.history[1].get("tool_call_id") == "tc_1"
def test_assistant_reasoning_fields_persisted(self, manager):
"""ACP session restore should preserve assistant reasoning context."""
state = manager.create_session()
state.history.append({
"role": "assistant",
"content": "hello",
"reasoning": "step-by-step",
"reasoning_details": [
{"type": "thinking", "thinking": "first thought"},
],
"codex_reasoning_items": [
{"type": "reasoning", "id": "rs_123", "encrypted_content": "enc_blob"},
],
})
manager.save_session(state.session_id)
with manager._lock:
del manager._sessions[state.session_id]
restored = manager.get_session(state.session_id)
assert restored is not None
assert restored.history == [{
"role": "assistant",
"content": "hello",
"reasoning": "step-by-step",
"reasoning_details": [
{"type": "thinking", "thinking": "first thought"},
],
"codex_reasoning_items": [
{"type": "reasoning", "id": "rs_123", "encrypted_content": "enc_blob"},
],
}]
def test_restore_preserves_persisted_provider_snapshot(self, tmp_path, monkeypatch):
"""Restored ACP sessions should keep their original runtime provider."""
runtime_choice = {"provider": "anthropic"}
def fake_resolve_runtime_provider(requested=None, **kwargs):
provider = requested or runtime_choice["provider"]
return {
"provider": provider,
"api_mode": "anthropic_messages" if provider == "anthropic" else "chat_completions",
"base_url": f"https://{provider}.example/v1",
"api_key": f"{provider}-key",
"command": None,
"args": [],
}
def fake_agent(**kwargs):
return SimpleNamespace(
model=kwargs.get("model"),
provider=kwargs.get("provider"),
base_url=kwargs.get("base_url"),
api_mode=kwargs.get("api_mode"),
)
monkeypatch.setattr("hermes_cli.config.load_config", lambda: {
"model": {"provider": runtime_choice["provider"], "default": "test-model"}
})
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
fake_resolve_runtime_provider,
)
db = SessionDB(tmp_path / "state.db")
with patch("run_agent.AIAgent", side_effect=fake_agent):
manager = SessionManager(db=db)
state = manager.create_session(cwd="/work")
manager.save_session(state.session_id)
with manager._lock:
del manager._sessions[state.session_id]
runtime_choice["provider"] = "openrouter"
restored = manager.get_session(state.session_id)
assert restored is not None
assert restored.agent.provider == "anthropic"
assert restored.agent.base_url == "https://anthropic.example/v1"
def test_acp_agents_route_human_output_to_stderr(self, tmp_path, monkeypatch):
"""ACP agents must keep stdout clean for JSON-RPC stdio transport."""
def fake_resolve_runtime_provider(requested=None, **kwargs):
return {
"provider": "openrouter",
"api_mode": "chat_completions",
"base_url": "https://openrouter.example/v1",
"api_key": "test-key",
"command": None,
"args": [],
}
def fake_agent(**kwargs):
return SimpleNamespace(model=kwargs.get("model"), _print_fn=None)
monkeypatch.setattr("hermes_cli.config.load_config", lambda: {
"model": {"provider": "openrouter", "default": "test-model"}
})
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
fake_resolve_runtime_provider,
)
db = SessionDB(tmp_path / "state.db")
with patch("run_agent.AIAgent", side_effect=fake_agent):
manager = SessionManager(db=db)
state = manager.create_session(cwd="/work")
stdout_buf = io.StringIO()
stderr_buf = io.StringIO()
with contextlib.redirect_stdout(stdout_buf), contextlib.redirect_stderr(stderr_buf):
state.agent._print_fn("ACP noise")
assert stdout_buf.getvalue() == ""
assert stderr_buf.getvalue() == "ACP noise\n"
+638
View File
@@ -0,0 +1,638 @@
"""Tests for acp_adapter.tools — tool kind mapping and ACP content building."""
from acp_adapter.edit_approval import EditProposal
from acp_adapter.tools import (
TOOL_KIND_MAP,
build_tool_complete,
build_tool_start,
build_tool_title,
extract_locations,
get_tool_kind,
make_tool_call_id,
)
from acp.schema import (
FileEditToolCallContent,
ContentToolCallContent,
ToolCallLocation,
ToolCallStart,
ToolCallProgress,
)
# ---------------------------------------------------------------------------
# TOOL_KIND_MAP coverage
# ---------------------------------------------------------------------------
COMMON_HERMES_TOOLS = ["read_file", "search_files", "terminal", "patch", "write_file", "process"]
class TestToolKindMap:
def test_all_hermes_tools_have_kind(self):
"""Every common hermes tool should appear in TOOL_KIND_MAP."""
for tool in COMMON_HERMES_TOOLS:
assert tool in TOOL_KIND_MAP, f"{tool} missing from TOOL_KIND_MAP"
def test_tool_kind_read_file(self):
assert get_tool_kind("read_file") == "read"
def test_tool_kind_terminal(self):
assert get_tool_kind("terminal") == "execute"
def test_tool_kind_patch(self):
assert get_tool_kind("patch") == "edit"
def test_tool_kind_write_file(self):
assert get_tool_kind("write_file") == "edit"
def test_tool_kind_web_search(self):
assert get_tool_kind("web_search") == "fetch"
def test_tool_kind_execute_code(self):
assert get_tool_kind("execute_code") == "execute"
def test_tool_kind_todo(self):
assert get_tool_kind("todo") == "other"
def test_tool_kind_skill_view(self):
assert get_tool_kind("skill_view") == "read"
def test_tool_kind_browser_navigate(self):
assert get_tool_kind("browser_navigate") == "fetch"
def test_unknown_tool_returns_other_kind(self):
assert get_tool_kind("nonexistent_tool_xyz") == "other"
# ---------------------------------------------------------------------------
# make_tool_call_id
# ---------------------------------------------------------------------------
class TestMakeToolCallId:
def test_returns_string(self):
tc_id = make_tool_call_id()
assert isinstance(tc_id, str)
def test_starts_with_tc_prefix(self):
tc_id = make_tool_call_id()
assert tc_id.startswith("tc-")
def test_ids_are_unique(self):
ids = {make_tool_call_id() for _ in range(100)}
assert len(ids) == 100
# ---------------------------------------------------------------------------
# build_tool_title
# ---------------------------------------------------------------------------
class TestBuildToolTitle:
def test_terminal_title_includes_command(self):
title = build_tool_title("terminal", {"command": "ls -la /tmp"})
assert "ls -la /tmp" in title
def test_terminal_title_truncates_long_command(self):
long_cmd = "x" * 200
title = build_tool_title("terminal", {"command": long_cmd})
assert len(title) < 120
assert "..." in title
def test_read_file_title(self):
title = build_tool_title("read_file", {"path": "/etc/hosts"})
assert "/etc/hosts" in title
def test_patch_title(self):
title = build_tool_title("patch", {"path": "main.py", "mode": "replace"})
assert "main.py" in title
def test_search_title(self):
title = build_tool_title("search_files", {"pattern": "TODO"})
assert "TODO" in title
def test_web_search_title(self):
title = build_tool_title("web_search", {"query": "python asyncio"})
assert "python asyncio" in title
def test_skill_view_title_includes_skill_name(self):
title = build_tool_title("skill_view", {"name": "github-pitfalls"})
assert title == "skill view (github-pitfalls)"
def test_skill_view_title_includes_linked_file(self):
title = build_tool_title("skill_view", {"name": "github-pitfalls", "file_path": "references/api.md"})
assert title == "skill view (github-pitfalls/references/api.md)"
def test_execute_code_title_includes_first_code_line(self):
title = build_tool_title("execute_code", {"code": "\nfrom hermes_tools import terminal\nprint('done')"})
assert title == "python: from hermes_tools import terminal"
def test_skill_manage_title_includes_action_and_target(self):
title = build_tool_title(
"skill_manage",
{"action": "patch", "name": "hermes-agent-operations", "file_path": "references/acp.md"},
)
assert title == "skill patch: hermes-agent-operations/references/acp.md"
def test_unknown_tool_uses_name(self):
title = build_tool_title("some_new_tool", {"foo": "bar"})
assert title == "some_new_tool"
# ---------------------------------------------------------------------------
# build_tool_start
# ---------------------------------------------------------------------------
class TestBuildToolStart:
def test_build_tool_start_for_patch(self):
"""patch start should not duplicate the edit-approval diff."""
args = {
"path": "src/main.py",
"old_string": "print('hello')",
"new_string": "print('world')",
}
result = build_tool_start("tc-1", "patch", args)
assert isinstance(result, ToolCallStart)
assert result.kind == "edit"
assert len(result.content) >= 1
item = result.content[0]
assert isinstance(item, ContentToolCallContent)
assert "Approval prompt shows the diff" in item.content.text
assert "src/main.py" in item.content.text
def test_build_tool_start_for_write_file(self):
"""write_file start should not duplicate the edit-approval diff."""
args = {"path": "new_file.py", "content": "print('hello')"}
result = build_tool_start("tc-w1", "write_file", args)
assert isinstance(result, ToolCallStart)
assert result.kind == "edit"
assert len(result.content) >= 1
item = result.content[0]
assert isinstance(item, ContentToolCallContent)
assert "Approval prompt shows the diff" in item.content.text
assert "new_file.py" in item.content.text
def test_auto_approved_edit_start_shows_diff_content(self):
"""Auto-approved edit starts need the diff because no approval card exists."""
args = {"path": "/tmp/acp.txt", "old_string": "old", "new_string": "new"}
result = build_tool_start(
"tc-auto-edit",
"patch",
args,
edit_diff=EditProposal("patch", "/tmp/acp.txt", "old\n", "new\n", args),
)
assert isinstance(result, ToolCallStart)
assert result.kind == "edit"
assert len(result.content) == 1
item = result.content[0]
assert isinstance(item, FileEditToolCallContent)
assert item.path == "/tmp/acp.txt"
assert item.old_text == "old\n"
assert item.new_text == "new\n"
def test_build_tool_start_for_terminal(self):
"""terminal should produce text content with the command."""
args = {"command": "ls -la /tmp"}
result = build_tool_start("tc-2", "terminal", args)
assert isinstance(result, ToolCallStart)
assert result.kind == "execute"
assert len(result.content) >= 1
content_item = result.content[0]
assert isinstance(content_item, ContentToolCallContent)
# The wrapped text block should contain the command
text = content_item.content.text
assert "ls -la /tmp" in text
def test_build_tool_start_for_read_file(self):
"""read_file start should stay compact; completion carries file contents."""
args = {"path": "/etc/hosts", "offset": 1, "limit": 50}
result = build_tool_start("tc-3", "read_file", args)
assert isinstance(result, ToolCallStart)
assert result.kind == "read"
assert result.content is None
assert result.raw_input is None
def test_build_tool_start_for_web_extract_is_compact(self):
"""web_extract start should stay compact; title identifies URLs."""
args = {"urls": ["https://example.com/docs"]}
result = build_tool_start("tc-web-start", "web_extract", args)
assert isinstance(result, ToolCallStart)
assert result.title == "extract: https://example.com/docs"
assert result.kind == "fetch"
assert result.content is None
assert result.raw_input is None
def test_build_tool_start_for_browser_navigate(self):
"""browser_navigate should emit a polished start event."""
args = {"url": "https://x.com"}
result = build_tool_start("tc-browser-start", "browser_navigate", args)
assert isinstance(result, ToolCallStart)
assert result.title == "navigate: https://x.com"
assert result.kind == "fetch"
assert result.content[0].content.text == '{\n "url": "https://x.com"\n}'
assert result.raw_input is None
def test_build_tool_start_for_search(self):
"""search_files should include pattern in content."""
args = {"pattern": "TODO", "target": "content"}
result = build_tool_start("tc-4", "search_files", args)
assert isinstance(result, ToolCallStart)
assert result.kind == "search"
assert "TODO" in result.content[0].content.text
assert result.raw_input is None
def test_build_tool_start_for_todo_is_human_readable(self):
args = {"todos": [{"id": "one", "content": "Fix ACP rendering", "status": "in_progress"}]}
result = build_tool_start("tc-todo", "todo", args)
assert result.title == "todo (1 item)"
assert "Fix ACP rendering" in result.content[0].content.text
assert result.raw_input is None
def test_build_tool_start_for_skill_view_is_human_readable(self):
result = build_tool_start("tc-skill", "skill_view", {"name": "github-pitfalls"})
assert result.title == "skill view (github-pitfalls)"
assert "github-pitfalls" in result.content[0].content.text
assert result.raw_input is None
def test_build_tool_start_for_execute_code_shows_code_preview(self):
result = build_tool_start("tc-code", "execute_code", {"code": "print('hello')"})
assert result.kind == "execute"
assert result.title == "python: print('hello')"
assert "```python" in result.content[0].content.text
assert "print('hello')" in result.content[0].content.text
assert result.raw_input is None
def test_build_tool_start_for_skill_manage_patch_shows_diff(self):
result = build_tool_start(
"tc-skill-manage",
"skill_manage",
{
"action": "patch",
"name": "hermes-agent-operations",
"file_path": "references/acp.md",
"old_string": "old advice",
"new_string": "new advice",
},
)
assert result.kind == "edit"
assert result.title == "skill patch: hermes-agent-operations/references/acp.md"
assert isinstance(result.content[0], FileEditToolCallContent)
assert result.content[0].path == "skills/hermes-agent-operations/references/acp.md"
assert result.content[0].old_text == "old advice"
assert result.content[0].new_text == "new advice"
assert result.raw_input is None
def test_build_tool_start_generic_fallback(self):
"""Unknown tools should get a generic text representation."""
args = {"foo": "bar", "baz": 42}
result = build_tool_start("tc-5", "some_tool", args)
assert isinstance(result, ToolCallStart)
assert result.kind == "other"
# ---------------------------------------------------------------------------
# build_tool_complete
# ---------------------------------------------------------------------------
class TestBuildToolComplete:
def test_build_tool_complete_for_terminal(self):
"""Completed terminal call should include output text."""
result = build_tool_complete("tc-2", "terminal", "total 42\ndrwxr-xr-x 2 root root 4096 ...")
assert isinstance(result, ToolCallProgress)
assert result.status == "completed"
assert len(result.content) >= 1
content_item = result.content[0]
assert isinstance(content_item, ContentToolCallContent)
assert "total 42" in content_item.content.text
assert result.raw_output is None
def test_build_tool_complete_for_todo_is_checklist(self):
result = build_tool_complete(
"tc-todo",
"todo",
'{"todos":[{"id":"a","content":"Inspect ACP","status":"completed"},{"id":"b","content":"Patch renderers","status":"in_progress"}],"summary":{"total":2,"pending":0,"in_progress":1,"completed":1,"cancelled":0}}',
)
text = result.content[0].content.text
assert "✅ Inspect ACP" in text
assert "- 🔄 Patch renderers" in text
assert "**Progress:** 1 completed, 1 in progress, 0 pending" in text
assert result.raw_output is None
def test_build_tool_complete_for_skill_view_summarizes_content_without_raw_json(self):
result = build_tool_complete(
"tc-skill",
"skill_view",
'{"success":true,"name":"github-pitfalls","description":"GitHub gotchas","content":"# GitHub Pitfalls\\nUse gh carefully.","path":"github/github-pitfalls/SKILL.md"}',
)
text = result.content[0].content.text
assert "**Skill loaded**" in text
assert "`github-pitfalls`" in text
assert "GitHub gotchas" in text
assert "GitHub Pitfalls" in text
assert "Use gh carefully" not in text
assert "Full skill content is available to the agent" in text
assert result.raw_output is None
def test_build_tool_complete_for_execute_code_formats_output(self):
result = build_tool_complete("tc-code", "execute_code", '{"output":"hello\\n","exit_code":0}')
text = result.content[0].content.text
assert "Exit code: 0" in text
assert "hello" in text
assert result.raw_output is None
def test_build_tool_complete_marks_success_false_as_failed(self):
result = build_tool_complete("tc-fail", "skill_manage", '{"success": false, "error": "boom"}')
assert result.status == "failed"
def test_build_tool_complete_marks_ok_false_as_failed(self):
result = build_tool_complete("tc-fail", "some_tool", '{"ok": false, "error": "boom"}')
assert result.status == "failed"
def test_build_tool_complete_marks_exit_code_nonzero_as_failed(self):
result = build_tool_complete("tc-fail", "terminal", '{"output": "bad", "exit_code": 2}')
assert result.status == "failed"
def test_build_tool_complete_marks_returncode_nonzero_as_failed(self):
result = build_tool_complete("tc-fail", "execute_code", '{"output": "bad", "returncode": 2}')
assert result.status == "failed"
def test_build_tool_complete_keeps_plain_error_text_completed(self):
result = build_tool_complete("tc-ok", "terminal", "tests failed: 1 assertion error")
assert result.status == "completed"
def test_build_tool_complete_marks_raised_exception_prefix_as_failed(self):
"""The agent's tool executor wraps raised exceptions in a canonical
"Error executing tool '<name>': ..." prefix. That prefix is unique to
the wrapper and means the tool blew up, so it must surface as failed
in Zed regardless of whether the body parses as JSON.
"""
result = build_tool_complete(
"tc-fail-exc",
"patch",
"Error executing tool 'patch': KeyError: 'foo'",
)
assert result.status == "failed"
def test_build_tool_complete_does_not_match_error_word_alone(self):
"""Bare 'Error: ...' messages (without the unique 'Error executing
tool '<name>':' prefix) must still be reported as completed — they
legitimately appear in compiler/linter/test output.
"""
result = build_tool_complete(
"tc-ok-error-word",
"terminal",
"Error: pytest collected 0 items",
)
assert result.status == "completed"
def test_build_tool_complete_marks_structured_polished_tool_error_as_failed(self):
result = build_tool_complete("tc-fail", "read_file", '{"error": "File not found"}')
assert result.status == "failed"
def test_build_tool_complete_keeps_json_error_without_failure_flag_completed(self):
result = build_tool_complete("tc-ok", "some_tool", '{"error": "timeout while reading optional source"}')
assert result.status == "completed"
def test_build_tool_complete_for_skill_manage_summarizes_without_raw_json(self):
result = build_tool_complete(
"tc-skill-manage",
"skill_manage",
'{"success":true,"message":"Patched references/hermes-acp-zed-rendering.md in skill \'hermes-agent-operations\' (1 replacement)."}',
function_args={
"action": "patch",
"name": "hermes-agent-operations",
"file_path": "references/hermes-acp-zed-rendering.md",
},
)
text = result.content[0].content.text
assert "**✅ Skill updated**" in text
assert "`patch`" in text
assert "`hermes-agent-operations`" in text
assert "references/hermes-acp-zed-rendering.md" in text
assert "{\"success\"" not in text
assert result.raw_output is None
def test_build_tool_complete_for_read_file_formats_content(self):
result = build_tool_complete(
"tc-read",
"read_file",
'{"content":"1|hello\\n2|world","total_lines":2}',
function_args={"path":"README.md","offset":1,"limit":20},
)
text = result.content[0].content.text
assert "Read README.md" in text
assert "```\n1|hello\n2|world\n```" in text
assert result.raw_output is None
def test_build_tool_complete_for_search_files_formats_matches(self):
result = build_tool_complete(
"tc-search",
"search_files",
'{"total_count":2,"matches":[{"path":"README.md","line":3,"content":"TODO: fix this"},{"path":"src/app.py","line":9,"content":"needle"}],"truncated":true}\n\n[Hint: Results truncated. Use offset=12 to see more.]',
)
text = result.content[0].content.text
assert "Search results" in text
assert "Found 2 matches" in text
assert "README.md:3" in text
assert "TODO: fix this" in text
assert "Results truncated" in text
assert result.raw_output is None
def test_build_tool_complete_for_process_list_formats_table(self):
result = build_tool_complete(
"tc-process",
"process",
'{"processes":[{"session_id":"p1","status":"running","pid":123,"command":"npm run dev"}]}',
function_args={"action":"list"},
)
text = result.content[0].content.text
assert "Processes: 1" in text
assert "`p1`" in text
assert "npm run dev" in text
assert result.raw_output is None
def test_build_tool_complete_for_delegate_task_summarizes_children(self):
result = build_tool_complete(
"tc-delegate",
"delegate_task",
'{"results":[{"task_index":0,"status":"completed","summary":"Reviewed ACP rendering.","model":"gpt-5.5","duration_seconds":3.2,"tool_trace":[{"tool":"read_file"}]}],"total_duration_seconds":3.4}',
)
text = result.content[0].content.text
assert "Delegation results: 1 task" in text
assert "Reviewed ACP rendering" in text
assert "gpt-5.5" in text
assert "Tools: read_file" in text
assert result.raw_output is None
def test_build_tool_complete_for_session_search_recent(self):
result = build_tool_complete(
"tc-session",
"session_search",
'{"success":true,"mode":"recent","results":[{"session_id":"s1","title":"ACP work","last_active":"2026-05-02","message_count":12,"preview":"Polished tool rendering."}],"count":1}',
)
text = result.content[0].content.text
assert "Recent sessions" in text
assert "ACP work" in text
assert "Polished tool rendering" in text
assert result.raw_output is None
def test_build_tool_complete_for_memory_avoids_dumping_entries(self):
result = build_tool_complete(
"tc-memory",
"memory",
'{"success":true,"target":"user","entries":["private long memory"],"usage":"1% — 19/2000 chars","entry_count":1,"message":"Entry added."}',
function_args={"action":"add","target":"user","content":"User likes concise ACP rendering."},
)
text = result.content[0].content.text
assert "Memory add saved" in text
assert "User likes concise ACP rendering" in text
assert "private long memory" not in text
assert result.raw_output is None
def test_build_tool_complete_for_web_extract_success_stays_compact(self):
result = build_tool_complete(
"tc-web-extract",
"web_extract",
'{"results":[{"url":"https://example.com","title":"Example","content":"# Intro\\nThis is extracted content."}]}',
)
assert result.content is None
assert result.raw_output is None
def test_build_tool_complete_for_web_extract_error_shows_error(self):
result = build_tool_complete(
"tc-web-extract-error",
"web_extract",
'{"results":[{"url":"https://example.com","title":"Example","error":"timeout"}]}',
)
text = result.content[0].content.text
assert "Web extract failed" in text
assert "https://example.com" in text
assert "timeout" in text
assert result.raw_output is None
def test_build_tool_complete_generically_formats_unknown_json_dict_without_raw_output(self):
result = build_tool_complete(
"tc-recall-search",
"memory_archive_search",
'{"results":[{"id":"obs-1","status":"active","content":"Recall should render as a readable summary."}],"trust":"lower-trust archive evidence"}',
)
text = result.content[0].content.text
assert "memory_archive_search result" in text
assert "lower-trust archive evidence" in text
assert "Recall should render as a readable summary" in text
assert "{\"results\"" not in text
assert result.raw_output is None
def test_build_tool_complete_generically_formats_unknown_json_list_without_raw_output(self):
result = build_tool_complete(
"tc-plugin-list",
"some_plugin_tool",
'[{"name":"alpha","status":"ok"},{"name":"beta","status":"ok"}]',
)
text = result.content[0].content.text
assert "some_plugin_tool: 2 items" in text
assert "alpha" in text
assert result.raw_output is None
def test_build_tool_complete_generically_formats_nested_json_without_inline_blob(self):
result = build_tool_complete(
"tc-recall-stats",
"memory_archive_stats",
'{"observations_by_status":{"active":12,"rejected":83},"capabilities":["sqlite-fts5-archive","hash-chain-audit"],"audit":{"ok":true,"count":208,"head":"abc123"}}',
)
text = result.content[0].content.text
assert "**observations_by_status:**" in text
assert "**active:** 12" in text
assert "**rejected:** 83" in text
assert "**capabilities:** 2 items" in text
assert "sqlite-fts5-archive" in text
assert "**audit:**" in text
assert "**ok:** True" in text
assert "{\"active\"" not in text
assert "[\"sqlite" not in text
assert result.raw_output is None
def test_build_tool_complete_for_search_files_files_only_formats_file_list(self):
result = build_tool_complete(
"tc-search-files",
"search_files",
'{"total_count":36,"files":["/home/nour/.hermes/config.yaml","/home/nour/.hermes/profiles/recall-test/config.yaml"],"truncated":true}',
)
text = result.content[0].content.text
assert "File search results" in text
assert "Found 36 files; showing 2." in text
assert "/home/nour/.hermes/config.yaml" in text
assert "use offset to page" in text
assert "{\"total_count\"" not in text
assert result.raw_output is None
def test_build_tool_complete_truncates_large_output(self):
"""Very large outputs should be truncated."""
big_output = "x" * 10000
result = build_tool_complete("tc-6", "read_file", big_output)
assert isinstance(result, ToolCallProgress)
display_text = result.content[0].content.text
assert len(display_text) < 6000
assert "truncated" in display_text
def test_build_tool_complete_for_patch_summarizes_without_repeating_diff(self):
"""Completed patch calls should not duplicate the edit-approval diff."""
patch_result = (
'{"success": true, "diff": "--- a/README.md\\n+++ b/README.md\\n@@ -1 +1,2 @@\\n old line\\n+new line\\n", '
'"files_modified": ["README.md"]}'
)
result = build_tool_complete("tc-p1", "patch", patch_result)
assert isinstance(result, ToolCallProgress)
assert len(result.content) == 1
item = result.content[0]
assert isinstance(item, ContentToolCallContent)
assert "✅ patch completed" in item.content.text
assert "README.md" in item.content.text
def test_build_tool_complete_for_patch_falls_back_to_text_when_no_diff(self):
result = build_tool_complete("tc-p2", "patch", '{"success": true}')
assert isinstance(result, ToolCallProgress)
assert isinstance(result.content[0], ContentToolCallContent)
def test_build_tool_complete_for_write_file_summarizes_without_repeating_diff(self, tmp_path):
target = tmp_path / "diff-test.txt"
snapshot = type("Snapshot", (), {"paths": [target], "before": {str(target): None}})()
target.write_text("hello from hermes\n", encoding="utf-8")
result = build_tool_complete(
"tc-wf1",
"write_file",
'{"bytes_written": 18, "dirs_created": false}',
function_args={"path": str(target), "content": "hello from hermes\n"},
snapshot=snapshot,
)
assert isinstance(result, ToolCallProgress)
assert len(result.content) == 1
item = result.content[0]
assert isinstance(item, ContentToolCallContent)
assert "✅ write_file completed" in item.content.text
assert "diff-test.txt" in item.content.text
# ---------------------------------------------------------------------------
# extract_locations
# ---------------------------------------------------------------------------
class TestExtractLocations:
def test_extract_locations_with_path(self):
args = {"path": "src/app.py", "offset": 42}
locs = extract_locations(args)
assert len(locs) == 1
assert isinstance(locs[0], ToolCallLocation)
assert locs[0].path == "src/app.py"
assert locs[0].line == 42
def test_extract_locations_without_path(self):
args = {"command": "echo hi"}
locs = extract_locations(args)
assert locs == []
+198
View File
@@ -0,0 +1,198 @@
import sys
from types import ModuleType, SimpleNamespace
import pytest
from acp.schema import TextContentBlock
from acp_adapter.server import HermesACPAgent
from acp_adapter.session import SessionManager
class FakeAgent:
def __init__(self):
self.model = "fake-model"
self.provider = "fake-provider"
self.enabled_toolsets = ["hermes-acp"]
self.disabled_toolsets = []
self.tools = []
self.valid_tool_names = set()
self.steers = []
self.runs = []
def steer(self, text):
self.steers.append(text)
return True
def run_conversation(self, *, user_message, conversation_history, task_id, **kwargs):
self.runs.append(user_message)
messages = list(conversation_history or [])
messages.append({"role": "user", "content": user_message})
final = f"ran: {user_message}"
messages.append({"role": "assistant", "content": final})
return {"final_response": final, "messages": messages}
class CaptureConn:
def __init__(self):
self.updates = []
async def session_update(self, *args, **kwargs):
if kwargs:
self.updates.append((kwargs.get("session_id"), kwargs.get("update")))
else:
self.updates.append((args[0], args[1]))
async def request_permission(self, *args, **kwargs):
return SimpleNamespace(outcome="allow")
class NoopDb:
def get_session(self, *_args, **_kwargs):
return None
def create_session(self, *_args, **_kwargs):
return None
def update_session(self, *_args, **_kwargs):
return None
def make_agent_and_state():
fake = FakeAgent()
manager = SessionManager(agent_factory=lambda **kwargs: fake, db=NoopDb())
acp_agent = HermesACPAgent(session_manager=manager)
state = manager.create_session(cwd=".")
conn = CaptureConn()
acp_agent.on_connect(conn)
return acp_agent, state, fake, conn
def test_acp_real_agent_gets_session_db_for_recall(monkeypatch):
"""ACP sessions persist to SessionDB; recall must receive the same DB handle."""
captured = {}
sentinel_db = NoopDb()
class CapturingAgent(FakeAgent):
def __init__(self, **kwargs):
super().__init__()
captured.update(kwargs)
def mod(name, **attrs):
module = ModuleType(name)
for key, value in attrs.items():
setattr(module, key, value)
return module
monkeypatch.setitem(sys.modules, "run_agent", mod("run_agent", AIAgent=CapturingAgent))
monkeypatch.setitem(
sys.modules,
"hermes_cli.config",
mod("hermes_cli.config", load_config=lambda: {"model": {"default": "m", "provider": "p"}}),
)
monkeypatch.setitem(
sys.modules,
"hermes_cli.runtime_provider",
mod(
"hermes_cli.runtime_provider",
resolve_runtime_provider=lambda **_kwargs: {
"provider": "p",
"api_mode": "chat_completions",
"base_url": "u",
"api_key": "k",
"command": None,
"args": [],
},
),
)
manager = SessionManager(db=sentinel_db)
agent = manager._make_agent(session_id="acp-session", cwd=".")
assert isinstance(agent, CapturingAgent)
assert captured["session_db"] is sentinel_db
assert captured["platform"] == "acp"
assert captured["session_id"] == "acp-session"
@pytest.mark.asyncio
async def test_acp_steer_slash_command_injects_into_running_agent():
acp_agent, state, fake, _conn = make_agent_and_state()
state.is_running = True
response = await acp_agent.prompt(
session_id=state.session_id,
prompt=[TextContentBlock(type="text", text="/steer prefer the simpler fix")],
)
assert response.stop_reason == "end_turn"
assert fake.steers == ["prefer the simpler fix"]
assert fake.runs == []
@pytest.mark.asyncio
async def test_acp_steer_after_zed_interrupt_replays_interrupted_prompt_with_guidance():
acp_agent, state, fake, _conn = make_agent_and_state()
state.interrupted_prompt_text = "write hi to a text file"
response = await acp_agent.prompt(
session_id=state.session_id,
prompt=[TextContentBlock(type="text", text="/steer write HELLO instead")],
)
assert response.stop_reason == "end_turn"
assert fake.steers == []
assert fake.runs == [
"write hi to a text file\n\nUser correction/guidance after interrupt: write HELLO instead"
]
assert state.interrupted_prompt_text == ""
@pytest.mark.asyncio
async def test_acp_steer_on_idle_session_runs_as_regular_prompt():
# /steer on an idle session (no running turn, nothing to salvage) should
# run the steer payload as a normal user prompt — NOT silently append it
# to state.queued_prompts. Without this, users on Zed / other ACP clients
# see their /steer turn into "queued for the next turn" when they never
# typed /queue. Matches gateway/run.py ~L4898 idle-/steer behavior.
acp_agent, state, fake, _conn = make_agent_and_state()
response = await acp_agent.prompt(
session_id=state.session_id,
prompt=[TextContentBlock(type="text", text="/steer summarize the README")],
)
assert response.stop_reason == "end_turn"
assert fake.steers == []
assert fake.runs == ["summarize the README"]
assert state.queued_prompts == []
@pytest.mark.asyncio
async def test_acp_queue_slash_command_adds_next_turn_without_running_now():
acp_agent, state, fake, _conn = make_agent_and_state()
response = await acp_agent.prompt(
session_id=state.session_id,
prompt=[TextContentBlock(type="text", text="/queue run the tests after this")],
)
assert response.stop_reason == "end_turn"
assert state.queued_prompts == ["run the tests after this"]
assert fake.runs == []
@pytest.mark.asyncio
async def test_acp_prompt_drains_queued_turns_after_current_run():
acp_agent, state, fake, conn = make_agent_and_state()
state.queued_prompts.append("then run tests")
response = await acp_agent.prompt(
session_id=state.session_id,
prompt=[TextContentBlock(type="text", text="make the change")],
)
assert response.stop_reason == "end_turn"
assert fake.runs == ["make the change", "then run tests"]
assert state.queued_prompts == []
agent_messages = [u for _sid, u in conn.updates if getattr(u, "session_update", None) == "agent_message_chunk"]
assert len(agent_messages) >= 2
+159
View File
@@ -0,0 +1,159 @@
import base64
import pytest
from acp.schema import (
BlobResourceContents,
EmbeddedResourceContentBlock,
ImageContentBlock,
ResourceContentBlock,
TextContentBlock,
TextResourceContents,
)
from acp_adapter.server import HermesACPAgent, _content_blocks_to_openai_user_content
def test_acp_image_blocks_convert_to_openai_multimodal_content():
content = _content_blocks_to_openai_user_content([
TextContentBlock(type="text", text="What is in this image?"),
ImageContentBlock(type="image", data="aGVsbG8=", mimeType="image/png"),
])
assert content == [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,aGVsbG8="},
},
]
def test_text_only_acp_blocks_stay_string_for_legacy_prompt_path():
content = _content_blocks_to_openai_user_content([
TextContentBlock(type="text", text="/help"),
])
assert content == "/help"
def test_acp_resource_link_file_is_inlined_as_text(tmp_path):
attached = tmp_path / "notes.md"
attached.write_text("# Notes\n\nAttached file body", encoding="utf-8")
content = _content_blocks_to_openai_user_content([
TextContentBlock(type="text", text="Please read this file"),
ResourceContentBlock(
type="resource_link",
name="notes.md",
title="Project notes",
uri=attached.as_uri(),
mimeType="text/markdown",
),
])
assert content == (
"Please read this file\n"
"[Attached file: Project notes (notes.md)]\n"
f"URI: {attached.as_uri()}\n\n"
"# Notes\n\nAttached file body"
)
def test_acp_embedded_text_resource_is_inlined_as_text():
content = _content_blocks_to_openai_user_content([
EmbeddedResourceContentBlock(
type="resource",
resource=TextResourceContents(
uri="file:///workspace/todo.txt",
mimeType="text/plain",
text="first\nsecond",
),
),
])
assert content == (
"[Attached file: todo.txt]\n"
"URI: file:///workspace/todo.txt\n\n"
"first\nsecond"
)
@pytest.mark.asyncio
async def test_initialize_advertises_image_prompt_capability():
response = await HermesACPAgent().initialize()
assert response.agent_capabilities is not None
assert response.agent_capabilities.prompt_capabilities is not None
assert response.agent_capabilities.prompt_capabilities.image is True
# 1x1 transparent PNG — smallest valid image payload for inlining tests.
_ONE_PX_PNG = bytes.fromhex(
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000a49444154789c6300010000000500010d0a2db40000000049454e44ae426082"
)
def test_acp_resource_link_image_file_is_inlined_as_image_url(tmp_path):
attached = tmp_path / "shot.png"
attached.write_bytes(_ONE_PX_PNG)
content = _content_blocks_to_openai_user_content([
TextContentBlock(type="text", text="Look at this screenshot"),
ResourceContentBlock(
type="resource_link",
name="shot.png",
uri=attached.as_uri(),
mimeType="image/png",
),
])
assert isinstance(content, list)
# [user text, image header, image_url]
assert content[0] == {"type": "text", "text": "Look at this screenshot"}
assert content[1]["type"] == "text"
assert "[Attached image: shot.png]" in content[1]["text"]
assert content[2]["type"] == "image_url"
expected_url = "data:image/png;base64," + base64.b64encode(_ONE_PX_PNG).decode("ascii")
assert content[2]["image_url"]["url"] == expected_url
def test_acp_resource_link_image_mime_inferred_from_suffix(tmp_path):
"""No mimeType sent — should still be recognised as image by file suffix."""
attached = tmp_path / "pic.jpg"
attached.write_bytes(_ONE_PX_PNG) # content doesn't matter for the code path
content = _content_blocks_to_openai_user_content([
ResourceContentBlock(
type="resource_link",
name="pic.jpg",
uri=attached.as_uri(),
),
])
assert isinstance(content, list)
image_parts = [p for p in content if p.get("type") == "image_url"]
assert len(image_parts) == 1
assert image_parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
def test_acp_embedded_blob_image_is_inlined_as_image_url():
b64 = base64.b64encode(_ONE_PX_PNG).decode("ascii")
content = _content_blocks_to_openai_user_content([
EmbeddedResourceContentBlock(
type="resource",
resource=BlobResourceContents(
uri="file:///tmp/embed.png",
mimeType="image/png",
blob=b64,
),
),
])
assert isinstance(content, list)
assert content[0]["type"] == "text"
assert "[Attached image: embed.png]" in content[0]["text"]
assert content[1] == {
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{b64}"},
}
@@ -0,0 +1,87 @@
"""Regression tests for ACP adapter detection under Azure Foundry Entra ID.
The ACP adapter's ``detect_provider`` previously gated on
``isinstance(api_key, str)`` and returned ``None`` for any runtime that
returned a callable ``api_key`` — i.e. Azure Foundry with
``auth_mode=entra_id``. Downstream, ACP would default to
``"openrouter"`` and reject the legitimate provider in its auth handshake.
This test pins the callable-aware fix so it never regresses.
"""
from __future__ import annotations
from unittest.mock import patch
class TestDetectProviderEntra:
def test_callable_api_key_is_a_valid_credential(self):
"""A runtime returning a callable ``api_key`` (Entra bearer token
provider) must be detected as a configured provider, not
``None``."""
from acp_adapter import auth as _acp_auth
def _fake_runtime(**_kwargs):
return {
"provider": "azure-foundry",
"api_mode": "chat_completions",
"auth_mode": "entra_id",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_key": lambda: "jwt-fresh",
}
with patch(
"hermes_cli.runtime_provider.resolve_runtime_provider",
side_effect=_fake_runtime,
):
assert _acp_auth.detect_provider() == "azure-foundry"
assert _acp_auth.has_provider() is True
def test_string_api_key_still_works(self):
from acp_adapter import auth as _acp_auth
def _fake_runtime(**_kwargs):
return {
"provider": "openrouter",
"api_key": "sk-or-static-key",
}
with patch(
"hermes_cli.runtime_provider.resolve_runtime_provider",
side_effect=_fake_runtime,
):
assert _acp_auth.detect_provider() == "openrouter"
def test_empty_string_api_key_returns_none(self):
from acp_adapter import auth as _acp_auth
def _fake_runtime(**_kwargs):
return {"provider": "openrouter", "api_key": ""}
with patch(
"hermes_cli.runtime_provider.resolve_runtime_provider",
side_effect=_fake_runtime,
):
assert _acp_auth.detect_provider() is None
def test_missing_provider_returns_none(self):
"""A callable api_key without a provider is still ``None`` —
we don't synthesize a provider name from the credential shape."""
from acp_adapter import auth as _acp_auth
def _fake_runtime(**_kwargs):
return {"api_key": lambda: "jwt-fresh", "provider": ""}
with patch(
"hermes_cli.runtime_provider.resolve_runtime_provider",
side_effect=_fake_runtime,
):
assert _acp_auth.detect_provider() is None
def test_resolver_exception_returns_none(self):
from acp_adapter import auth as _acp_auth
with patch(
"hermes_cli.runtime_provider.resolve_runtime_provider",
side_effect=RuntimeError("simulated"),
):
assert _acp_auth.detect_provider() is None
View File
+1
View File
@@ -0,0 +1 @@
"""Pytest helpers for LSP-related tests."""
+159
View File
@@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""A minimal in-process LSP server used by tests.
Speaks just enough LSP to drive :class:`agent.lsp.client.LSPClient`
through a full lifecycle: ``initialize``, ``initialized``,
``textDocument/didOpen``, ``textDocument/didChange``, then a
``textDocument/publishDiagnostics`` notification followed by
``shutdown`` + ``exit``.
Behaviour (all behaviours selectable via env var ``MOCK_LSP_SCRIPT``):
- ``"clean"`` — initialize, accept didOpen/didChange, push empty
diagnostics on every open/change, exit cleanly on shutdown.
- ``"errors"`` — same as ``clean`` but the published diagnostics
carry one severity-1 entry pointing at line 0:0.
- ``"crash"`` — exit immediately after responding to ``initialize``
(simulates a crashing server).
- ``"slow"`` — same as ``clean`` but sleeps 1s before responding to
``initialize`` (lets us test timeout behaviour).
The script writes JSON-RPC framed messages to stdout and reads from
stdin. No third-party dependencies — uses only stdlib so it runs
under whatever Python the test process picks up.
"""
from __future__ import annotations
import json
import os
import sys
import time
def read_message():
"""Read one Content-Length framed JSON-RPC message from stdin."""
headers = {}
while True:
line = sys.stdin.buffer.readline()
if not line:
return None
line = line.rstrip(b"\r\n")
if not line:
break
k, _, v = line.decode("ascii").partition(":")
headers[k.strip().lower()] = v.strip()
n = int(headers["content-length"])
body = sys.stdin.buffer.read(n)
return json.loads(body.decode("utf-8"))
def write_message(obj):
body = json.dumps(obj, separators=(",", ":")).encode("utf-8")
sys.stdout.buffer.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii"))
sys.stdout.buffer.write(body)
sys.stdout.buffer.flush()
def main():
script = os.environ.get("MOCK_LSP_SCRIPT", "clean")
while True:
msg = read_message()
if msg is None:
return 0
if "id" in msg and msg.get("method") == "initialize":
if script == "slow":
time.sleep(1.0)
write_message(
{
"jsonrpc": "2.0",
"id": msg["id"],
"result": {
"capabilities": {
"textDocumentSync": 1, # Full
"diagnosticProvider": {"interFileDependencies": False, "workspaceDiagnostics": False},
},
"serverInfo": {"name": "mock-lsp", "version": "0.1"},
},
}
)
if script == "crash":
return 0
continue
if msg.get("method") == "initialized":
continue
if msg.get("method") == "workspace/didChangeConfiguration":
continue
if msg.get("method") == "workspace/didChangeWatchedFiles":
continue
if msg.get("method") in {"textDocument/didOpen", "textDocument/didChange"}:
params = msg.get("params") or {}
td = params.get("textDocument") or {}
uri = td.get("uri", "")
version = td.get("version", 0)
diagnostics = []
if script == "errors":
diagnostics = [
{
"range": {
"start": {"line": 0, "character": 0},
"end": {"line": 0, "character": 5},
},
"severity": 1,
"code": "MOCK001",
"source": "mock-lsp",
"message": "synthetic error from mock-lsp",
}
]
write_message(
{
"jsonrpc": "2.0",
"method": "textDocument/publishDiagnostics",
"params": {
"uri": uri,
"version": version,
"diagnostics": diagnostics,
},
}
)
continue
if msg.get("method") == "textDocument/diagnostic":
# Pull endpoint — return empty.
write_message(
{
"jsonrpc": "2.0",
"id": msg["id"],
"result": {"kind": "full", "items": []},
}
)
continue
if msg.get("method") == "textDocument/didSave":
continue
if msg.get("method") == "shutdown":
write_message({"jsonrpc": "2.0", "id": msg["id"], "result": None})
continue
if msg.get("method") == "exit":
return 0
# Unknown request: respond with method-not-found.
if "id" in msg:
write_message(
{
"jsonrpc": "2.0",
"id": msg["id"],
"error": {"code": -32601, "message": f"method not found: {msg.get('method')}"},
}
)
if __name__ == "__main__":
sys.exit(main())
+106
View File
@@ -0,0 +1,106 @@
"""Integration test: LSP layer is skipped on non-local backends.
The host-side LSP server can't see files inside a Docker/Modal/SSH
sandbox. When the agent's terminal env isn't ``LocalEnvironment``,
the file_operations layer must skip both ``snapshot_baseline`` and
``get_diagnostics_sync`` calls — falling back to the in-process
syntax check exactly as if LSP were disabled.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
from agent.lsp import eventlog
@pytest.fixture(autouse=True)
def _reset():
eventlog.reset_announce_caches()
def test_local_only_helper_returns_true_for_local_env():
from tools.environments.local import LocalEnvironment
from tools.file_operations import ShellFileOperations
fops = ShellFileOperations(LocalEnvironment(cwd="/tmp"))
assert fops._lsp_local_only() is True
def test_local_only_helper_returns_false_for_non_local_env():
"""A mocked non-local env (Docker/Modal/SSH stand-in) returns False."""
from tools.file_operations import ShellFileOperations
# Build something that's NOT a LocalEnvironment. We use a bare
# MagicMock — isinstance() against LocalEnvironment is False.
fake_env = MagicMock()
fake_env.execute = MagicMock(return_value=MagicMock(exit_code=0, stdout=""))
fake_env.cwd = "/sandbox"
fops = ShellFileOperations(fake_env)
assert fops._lsp_local_only() is False
def test_snapshot_baseline_skipped_for_non_local(monkeypatch):
"""Verify the LSP service's snapshot_baseline is NOT called when
the backend isn't local."""
from tools.file_operations import ShellFileOperations
fake_env = MagicMock()
fake_env.execute = MagicMock(return_value=MagicMock(exit_code=0, stdout=""))
fake_env.cwd = "/sandbox"
fops = ShellFileOperations(fake_env)
snapshot_called = []
class FakeService:
def snapshot_baseline(self, path):
snapshot_called.append(path)
monkeypatch.setattr("agent.lsp.get_service", lambda: FakeService())
fops._snapshot_lsp_baseline("/sandbox/x.py")
assert snapshot_called == [], "snapshot must be skipped for non-local backends"
def test_maybe_lsp_diagnostics_returns_empty_for_non_local(monkeypatch):
from tools.file_operations import ShellFileOperations
fake_env = MagicMock()
fake_env.execute = MagicMock(return_value=MagicMock(exit_code=0, stdout=""))
fake_env.cwd = "/sandbox"
fops = ShellFileOperations(fake_env)
called = []
class FakeService:
def enabled_for(self, path):
called.append(("enabled_for", path))
return True
def get_diagnostics_sync(self, path, **kw):
called.append(("get_diagnostics_sync", path))
return [{"severity": 1, "message": "should not see this"}]
monkeypatch.setattr("agent.lsp.get_service", lambda: FakeService())
result = fops._maybe_lsp_diagnostics("/sandbox/x.py")
assert result == ""
assert called == [], "service must not be queried for non-local backends"
def test_snapshot_baseline_called_for_local_env(tmp_path, monkeypatch):
from tools.environments.local import LocalEnvironment
from tools.file_operations import ShellFileOperations
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
snapshot_called = []
class FakeService:
def snapshot_baseline(self, path):
snapshot_called.append(path)
monkeypatch.setattr("agent.lsp.get_service", lambda: FakeService())
fops._snapshot_lsp_baseline(str(tmp_path / "x.py"))
assert snapshot_called == [str(tmp_path / "x.py")]
+210
View File
@@ -0,0 +1,210 @@
"""Tests for the broken-set short-circuit added to handle outer-timeout failures.
When ``snapshot_baseline`` or ``get_diagnostics_sync`` time out from the
service layer (because a language server hangs during initialize, or
the binary is wedged), the inner spawn task is cancelled — but the
inner exception handler that adds to ``_broken`` never runs. Without
the service-layer fallback added in this module, every subsequent
edit re-pays the full timeout cost until the process exits.
This module verifies:
- ``_mark_broken_for_file`` adds the right key
- ``enabled_for`` short-circuits on broken keys
- a missing binary is broken-set'd after one snapshot attempt
"""
from __future__ import annotations
from pathlib import Path
from unittest.mock import patch
import pytest
from agent.lsp.manager import LSPService
from agent.lsp.workspace import clear_cache
@pytest.fixture(autouse=True)
def _clear_workspace_cache():
clear_cache()
yield
clear_cache()
def _make_git_workspace(tmp_path: Path) -> Path:
"""Build a minimal git repo with a pyproject so pyright's root resolver fires."""
repo = tmp_path / "repo"
repo.mkdir()
(repo / ".git").mkdir()
(repo / "pyproject.toml").write_text("[project]\nname='t'\n")
return repo
def test_mark_broken_for_file_adds_correct_key(tmp_path, monkeypatch):
"""``_mark_broken_for_file`` keys the broken-set on
(server_id, per_server_root) so subsequent ``enabled_for`` calls
for files in the same project skip immediately."""
repo = _make_git_workspace(tmp_path)
monkeypatch.chdir(str(repo))
src = repo / "x.py"
src.write_text("")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
svc._mark_broken_for_file(str(src), RuntimeError("simulated"))
# The pyright server resolves to the repo root via pyproject.toml.
assert ("pyright", str(repo)) in svc._broken
finally:
svc.shutdown()
def test_enabled_for_returns_false_after_broken(tmp_path, monkeypatch):
"""Once a (server_id, root) pair is in the broken-set,
``enabled_for`` returns False so the file_operations layer skips
the LSP path entirely."""
repo = _make_git_workspace(tmp_path)
monkeypatch.chdir(str(repo))
src = repo / "x.py"
src.write_text("")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
# Initially enabled.
assert svc.enabled_for(str(src)) is True
# Mark broken.
svc._mark_broken_for_file(str(src), RuntimeError("simulated"))
# Now disabled — the broken-set short-circuits.
assert svc.enabled_for(str(src)) is False
finally:
svc.shutdown()
def test_enabled_for_other_file_in_same_project_also_skipped(tmp_path, monkeypatch):
"""The broken key is (server_id, root), so ALL files routed through
the same server in the same project are skipped — not just the one
that triggered the failure."""
repo = _make_git_workspace(tmp_path)
monkeypatch.chdir(str(repo))
a = repo / "a.py"
a.write_text("")
b = repo / "b.py"
b.write_text("")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
svc._mark_broken_for_file(str(a), RuntimeError("simulated"))
# Both files in the same project skip pyright now.
assert svc.enabled_for(str(a)) is False
assert svc.enabled_for(str(b)) is False
finally:
svc.shutdown()
def test_unrelated_project_not_affected_by_broken(tmp_path, monkeypatch):
"""Marking pyright broken for project A must NOT affect project B."""
repo_a = _make_git_workspace(tmp_path)
repo_b = tmp_path / "repo-b"
repo_b.mkdir()
(repo_b / ".git").mkdir()
(repo_b / "pyproject.toml").write_text("[project]\nname='b'\n")
a_src = repo_a / "x.py"
a_src.write_text("")
b_src = repo_b / "x.py"
b_src.write_text("")
monkeypatch.chdir(str(repo_a))
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
svc._mark_broken_for_file(str(a_src), RuntimeError("simulated"))
# Project A skipped.
assert svc.enabled_for(str(a_src)) is False
# Project B still enabled — the broken key is per-project.
monkeypatch.chdir(str(repo_b))
assert svc.enabled_for(str(b_src)) is True
finally:
svc.shutdown()
def test_mark_broken_handles_missing_server_silently(tmp_path):
"""If the file extension doesn't match any registered server,
``_mark_broken_for_file`` no-ops — nothing to mark."""
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
# No registered server for .xyz; must not raise.
svc._mark_broken_for_file(str(tmp_path / "weird.xyz"), RuntimeError("x"))
assert len(svc._broken) == 0
finally:
svc.shutdown()
def test_mark_broken_handles_no_workspace_silently(tmp_path):
"""File outside any git worktree → no workspace → no key to add."""
src = tmp_path / "orphan.py"
src.write_text("")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
svc._mark_broken_for_file(str(src), RuntimeError("x"))
assert len(svc._broken) == 0
finally:
svc.shutdown()
def test_snapshot_failure_marks_broken_via_outer_timeout(tmp_path, monkeypatch):
"""End-to-end: ``snapshot_baseline``'s outer ``_loop.run`` timeout
triggers ``_mark_broken_for_file``, so a second call to
``enabled_for`` returns False."""
repo = _make_git_workspace(tmp_path)
monkeypatch.chdir(str(repo))
src = repo / "x.py"
src.write_text("")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
try:
# Force the inner snapshot coroutine to raise.
async def boom(_path):
raise RuntimeError("outer-timeout simulated")
with patch.object(svc, "_snapshot_async", boom):
assert svc.enabled_for(str(src)) is True
svc.snapshot_baseline(str(src))
# After the failure, the file's pair is in the broken-set and
# ``enabled_for`` skips it.
assert ("pyright", str(repo)) in svc._broken
assert svc.enabled_for(str(src)) is False
finally:
svc.shutdown()
+143
View File
@@ -0,0 +1,143 @@
"""End-to-end client tests against the in-process mock LSP server.
Spins up :file:`_mock_lsp_server.py` as an actual subprocess, drives
it through real LSP traffic, and asserts diagnostic flow. This is
the closest thing we have to integration coverage without requiring
pyright/gopls/etc. to be installed in CI.
"""
from __future__ import annotations
import asyncio
import os
import sys
from pathlib import Path
import pytest
from agent.lsp.client import LSPClient
MOCK_SERVER = str(Path(__file__).parent / "_mock_lsp_server.py")
def _client(workspace: Path, script: str = "clean") -> LSPClient:
env = {"MOCK_LSP_SCRIPT": script, "PYTHONPATH": os.environ.get("PYTHONPATH", "")}
return LSPClient(
server_id=f"mock-{script}",
workspace_root=str(workspace),
command=[sys.executable, MOCK_SERVER],
env=env,
cwd=str(workspace),
)
@pytest.mark.asyncio
async def test_client_lifecycle_clean(tmp_path: Path):
"""Full lifecycle: spawn, initialize, open, get clean diagnostics, shutdown."""
f = tmp_path / "x.py"
f.write_text("print('hi')\n")
client = _client(tmp_path, "clean")
await client.start()
try:
assert client.is_running
version = await client.open_file(str(f), language_id="python")
assert version == 0
await client.wait_for_diagnostics(str(f), version, mode="document")
diags = client.diagnostics_for(str(f))
assert diags == []
finally:
await client.shutdown()
assert not client.is_running
@pytest.mark.asyncio
async def test_client_receives_published_errors(tmp_path: Path):
f = tmp_path / "x.py"
f.write_text("print('hi')\n")
client = _client(tmp_path, "errors")
await client.start()
try:
version = await client.open_file(str(f), language_id="python")
await client.wait_for_diagnostics(str(f), version, mode="document")
diags = client.diagnostics_for(str(f))
assert len(diags) == 1
d = diags[0]
assert d["severity"] == 1
assert d["code"] == "MOCK001"
assert d["source"] == "mock-lsp"
assert "synthetic error" in d["message"]
finally:
await client.shutdown()
@pytest.mark.asyncio
async def test_client_didchange_bumps_version(tmp_path: Path):
f = tmp_path / "x.py"
f.write_text("print('hi')\n")
client = _client(tmp_path, "errors")
await client.start()
try:
v0 = await client.open_file(str(f), language_id="python")
f.write_text("print('hi 2')\n")
v1 = await client.open_file(str(f), language_id="python") # re-open path = didChange
assert v1 == v0 + 1
await client.wait_for_diagnostics(str(f), v1, mode="document")
# Mock pushed a diagnostic for both events; merged view has one
# entry (push store keyed by file path).
diags = client.diagnostics_for(str(f))
assert len(diags) == 1
finally:
await client.shutdown()
@pytest.mark.asyncio
async def test_client_handles_crashing_server(tmp_path: Path):
"""When the server exits right after initialize, subsequent requests
fail gracefully (not hang)."""
f = tmp_path / "x.py"
f.write_text("")
client = _client(tmp_path, "crash")
await client.start() # should succeed (mock answers initialize before crashing)
# Give the OS a moment to deliver the EOF.
await asyncio.sleep(0.2)
# The reader loop should detect EOF and mark pending requests as failed.
try:
await asyncio.wait_for(
client.open_file(str(f), language_id="python"), timeout=2.0
)
except Exception:
pass # any exception is acceptable; the contract is "doesn't hang"
await client.shutdown()
@pytest.mark.asyncio
async def test_client_shutdown_idempotent(tmp_path: Path):
"""Calling shutdown twice must be safe."""
f = tmp_path / "x.py"
f.write_text("")
client = _client(tmp_path, "clean")
await client.start()
await client.shutdown()
await client.shutdown() # must not raise
@pytest.mark.asyncio
async def test_client_diagnostics_are_deduped(tmp_path: Path):
"""Repeated identical pushes must not produce duplicate diagnostics."""
f = tmp_path / "x.py"
f.write_text("")
client = _client(tmp_path, "errors")
await client.start()
try:
for _ in range(3):
v = await client.open_file(str(f), language_id="python")
await client.wait_for_diagnostics(str(f), v, mode="document")
diags = client.diagnostics_for(str(f))
# Push store overwrites on every notification — should have 1.
assert len(diags) == 1
finally:
await client.shutdown()
+262
View File
@@ -0,0 +1,262 @@
"""Tests for cross-edit LSP delta filtering.
The delta-filter contract spans three pieces:
1. ``agent.lsp.manager._diag_key`` — strict equality key including
the diagnostic's position range. Two diagnostics with the same
content but different lines are NOT equal under this key (they
are genuinely different diagnostics).
2. ``agent.lsp.range_shift.build_line_shift`` — derives a function
mapping pre-edit line numbers to post-edit line numbers from a
pre/post text pair.
3. ``agent.lsp.manager.LSPService.get_diagnostics_sync(line_shift=…)``
— applies the shift to baseline diagnostics before computing the
set-difference, so pre-existing errors at shifted lines hash
equal to their post-edit counterparts and get filtered out.
These tests exercise the contract at the unit level; the E2E case
(real LSP server, real shift) is covered in test_service.py.
"""
from __future__ import annotations
from agent.lsp.client import _diagnostic_key
from agent.lsp.manager import _diag_key
from agent.lsp.range_shift import (
build_line_shift,
shift_baseline,
shift_diagnostic_range,
)
def _diag(*, line: int, message: str = "Undefined variable",
severity: int = 1, code: str = "reportUndefinedVariable",
source: str = "Pyright", end_line: int | None = None) -> dict:
if end_line is None:
end_line = line
return {
"severity": severity,
"code": code,
"source": source,
"message": message,
"range": {
"start": {"line": line, "character": 0},
"end": {"line": end_line, "character": 10},
},
}
# ----------------------------------------------------------------------
# _diag_key: strict equality (with range)
# ----------------------------------------------------------------------
def test_diag_key_treats_shifted_diagnostics_as_distinct():
"""Two diagnostics with the same message but at different lines hash
differently — they are genuinely different diagnostics. The shift
map is what makes them equal AFTER remapping; the key itself stays
strict."""
a = _diag(line=100)
b = _diag(line=200)
assert _diag_key(a) != _diag_key(b)
def test_diag_key_matches_client_key_for_shifted_baseline():
"""When a baseline diagnostic is remapped through a shift, its
_diag_key must match the corresponding post-edit diagnostic's key
at the same coordinates. This is the contract the delta filter
relies on."""
pre = _diag(line=200)
# Edit deletes 14 lines above line 200, so the same error now
# appears at line 186 post-edit.
shift = lambda L: L - 14 if L >= 14 else L
shifted = shift_diagnostic_range(pre, shift)
assert shifted is not None
post = _diag(line=186)
assert _diag_key(shifted) == _diag_key(post)
def test_diag_key_distinguishes_message():
a = _diag(line=100, message="foo")
b = _diag(line=100, message="bar")
assert _diag_key(a) != _diag_key(b)
def test_diag_key_distinguishes_severity():
a = _diag(line=100, severity=1)
b = _diag(line=100, severity=2)
assert _diag_key(a) != _diag_key(b)
def test_diag_key_distinguishes_source():
a = _diag(line=100, source="Pyright")
b = _diag(line=100, source="Ruff")
assert _diag_key(a) != _diag_key(b)
def test_diag_key_matches_client_key_byte_for_byte():
"""The manager-side and client-side keys must agree on diagnostic
identity — they're used by two layers that need to round-trip the
same diagnostics through dedup and delta filtering."""
d = _diag(line=42)
assert _diag_key(d) == _diagnostic_key(d)
# ----------------------------------------------------------------------
# build_line_shift
# ----------------------------------------------------------------------
def test_shift_identity_for_identical_content():
shift = build_line_shift("a\nb\nc\n", "a\nb\nc\n")
assert shift(0) == 0
assert shift(1) == 1
assert shift(2) == 2
def test_shift_pure_deletion_above_line():
"""Delete 2 lines at the top; everything below shifts up by 2."""
pre = "line0\nline1\nline2\nline3\nline4\n"
post = "line2\nline3\nline4\n" # deleted lines 0-1
shift = build_line_shift(pre, post)
# Pre lines 0,1 → deleted → None
assert shift(0) is None
assert shift(1) is None
# Pre line 2 → post line 0
assert shift(2) == 0
# Pre line 4 → post line 2
assert shift(4) == 2
def test_shift_pure_insertion_above_line():
"""Insert 3 lines at the top; everything below shifts down by 3."""
pre = "line0\nline1\nline2\n"
post = "new0\nnew1\nnew2\nline0\nline1\nline2\n"
shift = build_line_shift(pre, post)
# Pre lines unchanged in identity, shifted by 3
assert shift(0) == 3
assert shift(1) == 4
assert shift(2) == 5
def test_shift_replacement_in_middle():
"""Replace 2 lines in the middle with 1 line. Lines above
unchanged; lines below shift up by 1."""
pre = "a\nb\nc\nd\ne\n"
post = "a\nb\nX\ne\n" # replaced lines 2,3 (c,d) with X
shift = build_line_shift(pre, post)
assert shift(0) == 0 # a → a
assert shift(1) == 1 # b → b
assert shift(2) is None # c → deleted
assert shift(3) is None # d → deleted
assert shift(4) == 3 # e → post line 3
def test_shift_handles_empty_pre():
"""First write of a file: pre is empty, post has content. Nothing
to shift, so the function should be well-defined for empty pre."""
shift = build_line_shift("", "hello\nworld\n")
# Any pre line falls past the end of an empty pre — anchor at end of post
assert shift(0) == 1
def test_shift_handles_empty_post():
"""File deleted to empty. Every pre line returns None."""
shift = build_line_shift("line0\nline1\n", "")
assert shift(0) is None
assert shift(1) is None
# ----------------------------------------------------------------------
# shift_diagnostic_range
# ----------------------------------------------------------------------
def test_shift_diag_remaps_start_and_end():
pre = "a\nb\nc\nd\n"
post = "X\na\nb\nc\nd\n" # one line inserted at top
shift = build_line_shift(pre, post)
d = _diag(line=2, end_line=2)
remapped = shift_diagnostic_range(d, shift)
assert remapped is not None
assert remapped["range"]["start"]["line"] == 3
assert remapped["range"]["end"]["line"] == 3
def test_shift_diag_drops_diagnostic_in_deleted_region():
pre = "a\nb\nc\nd\n"
post = "a\nd\n" # deleted lines 1,2 (b,c)
shift = build_line_shift(pre, post)
d = _diag(line=1)
assert shift_diagnostic_range(d, shift) is None
def test_shift_diag_does_not_mutate_original():
pre = "a\nb\n"
post = "X\na\nb\n"
shift = build_line_shift(pre, post)
d = _diag(line=0)
original_line = d["range"]["start"]["line"]
_ = shift_diagnostic_range(d, shift)
assert d["range"]["start"]["line"] == original_line
def test_shift_baseline_drops_deleted_and_remaps_rest():
pre = "a\nb\nc\nd\ne\n"
post = "a\ne\n" # deleted b,c,d
shift = build_line_shift(pre, post)
baseline = [
_diag(line=0, message="err on a"),
_diag(line=1, message="err on b"), # → deleted
_diag(line=2, message="err on c"), # → deleted
_diag(line=4, message="err on e"),
]
out = shift_baseline(baseline, shift)
assert [d["message"] for d in out] == ["err on a", "err on e"]
assert out[0]["range"]["start"]["line"] == 0
assert out[1]["range"]["start"]["line"] == 1
# ----------------------------------------------------------------------
# End-to-end: simulate the delta-filter pipeline
# ----------------------------------------------------------------------
def test_pipeline_filters_shifted_baseline_under_strict_key():
"""The exact scenario the bug fix is for: an edit deletes lines,
every diagnostic below shifts, and the delta filter (strict key
+ shifted baseline) correctly identifies them as pre-existing."""
pre = "line0\nline1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\n"
# Delete lines 2,3,4 — pre-existing errors at lines 7,8 should
# appear at lines 4,5 post-edit and be filtered out.
post = "line0\nline1\nline5\nline6\nline7\nline8\nline9\n"
shift = build_line_shift(pre, post)
baseline = [_diag(line=7, message="X"), _diag(line=8, message="Y")]
post_diags = [_diag(line=4, message="X"), _diag(line=5, message="Y")]
shifted_baseline = shift_baseline(baseline, shift)
seen = {_diag_key(d) for d in shifted_baseline}
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
# Both errors were pre-existing — filtered out.
assert new_diags == []
def test_pipeline_preserves_new_instance_at_different_line():
"""The case content-only keys would miss: the model introduces a
SECOND instance of the same error class at a new location. The
new instance must surface."""
pre = "good\ngood\ngood\n"
post = "good\nbad\ngood\nbad\n" # added 2 new error lines
shift = build_line_shift(pre, post)
baseline = [_diag(line=0, message="bad style")] # pre-existing
post_diags = [
_diag(line=0, message="bad style"), # pre-existing
_diag(line=1, message="bad style"), # NEW — different line
_diag(line=3, message="bad style"), # NEW — different line
]
shifted_baseline = shift_baseline(baseline, shift)
seen = {_diag_key(d) for d in shifted_baseline}
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
# Two genuinely new instances must be surfaced.
assert len(new_diags) == 2
assert {d["range"]["start"]["line"] for d in new_diags} == {1, 3}
+142
View File
@@ -0,0 +1,142 @@
"""Tests for the ``lsp_diagnostics`` field on WriteResult / PatchResult.
The field exists so the agent can read syntax errors (``lint``) and
semantic errors (``lsp_diagnostics``) as separate signals rather than
having LSP output prepended to the lint string.
"""
from __future__ import annotations
from unittest.mock import patch
from tools.environments.local import LocalEnvironment
from tools.file_operations import (
PatchResult,
ShellFileOperations,
WriteResult,
)
# ---------------------------------------------------------------------------
# Dataclass shape
# ---------------------------------------------------------------------------
def test_writeresult_lsp_diagnostics_optional():
r = WriteResult()
assert r.lsp_diagnostics is None
def test_writeresult_to_dict_omits_field_when_none():
r = WriteResult(bytes_written=10)
assert "lsp_diagnostics" not in r.to_dict()
def test_writeresult_to_dict_includes_field_when_set():
r = WriteResult(bytes_written=10, lsp_diagnostics="<diagnostics>...</diagnostics>")
d = r.to_dict()
assert d["lsp_diagnostics"] == "<diagnostics>...</diagnostics>"
def test_patchresult_to_dict_includes_field_when_set():
r = PatchResult(success=True, lsp_diagnostics="ERROR [1:1] thing")
d = r.to_dict()
assert d["lsp_diagnostics"] == "ERROR [1:1] thing"
def test_patchresult_to_dict_omits_field_when_none():
r = PatchResult(success=True)
assert "lsp_diagnostics" not in r.to_dict()
def test_patchresult_to_dict_omits_field_when_empty_string():
"""Empty string counts as falsy — agent shouldn't see an empty field."""
r = PatchResult(success=True, lsp_diagnostics="")
assert "lsp_diagnostics" not in r.to_dict()
# ---------------------------------------------------------------------------
# Channel separation: lint and lsp_diagnostics stay independent
# ---------------------------------------------------------------------------
def test_lint_and_lsp_diagnostics_are_separate_channels():
"""A WriteResult can carry BOTH a syntax-error lint AND an LSP
diagnostic block. They belong in separate fields."""
r = WriteResult(
bytes_written=42,
lint={"status": "error", "output": "SyntaxError: ..."},
lsp_diagnostics="<diagnostics>ERROR [1:5] type mismatch</diagnostics>",
)
d = r.to_dict()
assert "lint" in d
assert "lsp_diagnostics" in d
assert d["lint"]["output"] == "SyntaxError: ..."
assert "type mismatch" in d["lsp_diagnostics"]
# ---------------------------------------------------------------------------
# write_file populates the field via _maybe_lsp_diagnostics
# ---------------------------------------------------------------------------
def test_write_file_populates_lsp_diagnostics_when_layer_returns_block(tmp_path):
"""When the LSP layer returns a non-empty block, write_file puts it
into the ``lsp_diagnostics`` field — NOT into ``lint.output``."""
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
target = tmp_path / "x.py"
block = "<diagnostics file=\"x.py\">\nERROR [1:1] problem\n</diagnostics>"
with patch.object(fops, "_maybe_lsp_diagnostics", return_value=block):
res = fops.write_file(str(target), "x = 1\n")
assert res.lsp_diagnostics == block
# Lint is the syntax check, which is clean for "x = 1" — must NOT
# have the LSP block folded into it.
assert res.lint == {"status": "ok", "output": ""}
def test_write_file_lsp_diagnostics_none_when_layer_returns_empty(tmp_path):
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
target = tmp_path / "x.py"
with patch.object(fops, "_maybe_lsp_diagnostics", return_value=""):
res = fops.write_file(str(target), "x = 1\n")
assert res.lsp_diagnostics is None
def test_write_file_skips_lsp_when_syntax_failed(tmp_path):
"""If the syntax check finds errors, the LSP layer should not be
consulted (a file that won't parse won't yield meaningful semantic
diagnostics)."""
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
target = tmp_path / "broken.py"
with patch.object(fops, "_maybe_lsp_diagnostics") as mock_lsp:
res = fops.write_file(str(target), "def x(:\n") # syntax error
assert mock_lsp.call_count == 0
assert res.lsp_diagnostics is None
assert res.lint["status"] == "error"
# ---------------------------------------------------------------------------
# patch_replace propagates the field from the inner write_file
# ---------------------------------------------------------------------------
def test_patch_replace_propagates_lsp_diagnostics(tmp_path):
"""patch_replace's internal write_file populates lsp_diagnostics —
the outer PatchResult must carry it forward."""
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
target = tmp_path / "x.py"
target.write_text("x = 1\n")
block = "<diagnostics>ERROR [1:5] semantic issue</diagnostics>"
with patch.object(fops, "_maybe_lsp_diagnostics", return_value=block):
res = fops.patch_replace(str(target), "x = 1", "x = 2")
assert res.success is True
assert res.lsp_diagnostics == block
+199
View File
@@ -0,0 +1,199 @@
"""Tests for the structured logging dedup model.
The contract: a 1000-write session in one project should emit exactly
ONE INFO line ("active for <root>") at the default INFO threshold.
Steady-state events stay at DEBUG; first-time-seen events surface
once at INFO/WARNING.
"""
from __future__ import annotations
import logging
import pytest
from agent.lsp import eventlog
@pytest.fixture(autouse=True)
def _reset():
eventlog.reset_announce_caches()
yield
eventlog.reset_announce_caches()
@pytest.fixture
def caplog_lsp(caplog):
caplog.set_level(logging.DEBUG, logger="hermes.lint.lsp")
return caplog
# ---------------------------------------------------------------------------
# Steady-state silence (DEBUG)
# ---------------------------------------------------------------------------
def test_clean_emits_at_debug(caplog_lsp):
for _ in range(10):
eventlog.log_clean("pyright", "/proj/x.py")
info_records = [r for r in caplog_lsp.records if r.levelno >= logging.INFO]
debug_records = [r for r in caplog_lsp.records if r.levelno == logging.DEBUG]
assert info_records == []
assert len(debug_records) == 10
def test_disabled_emits_at_debug(caplog_lsp):
eventlog.log_disabled("pyright", "/x.py", "feature off")
eventlog.log_disabled("pyright", "/x.py", "ext not mapped")
assert all(r.levelno == logging.DEBUG for r in caplog_lsp.records)
# ---------------------------------------------------------------------------
# State transitions: INFO once, DEBUG thereafter
# ---------------------------------------------------------------------------
def test_active_for_fires_once_per_root(caplog_lsp):
for _ in range(50):
eventlog.log_active("pyright", "/proj")
info_records = [
r for r in caplog_lsp.records
if r.levelno == logging.INFO and "active for" in r.getMessage()
]
assert len(info_records) == 1
def test_active_for_fires_per_distinct_root(caplog_lsp):
eventlog.log_active("pyright", "/proj-a")
eventlog.log_active("pyright", "/proj-b")
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
assert len(info) == 2
def test_active_for_separate_per_server(caplog_lsp):
eventlog.log_active("pyright", "/proj")
eventlog.log_active("typescript", "/proj")
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
assert len(info) == 2
def test_no_project_root_fires_once_per_path(caplog_lsp):
for _ in range(5):
eventlog.log_no_project_root("pyright", "/orphan.py")
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
assert len(info) == 1
# ---------------------------------------------------------------------------
# Diagnostics events fire INFO every time
# ---------------------------------------------------------------------------
def test_diagnostics_always_info(caplog_lsp):
for i in range(5):
eventlog.log_diagnostics("pyright", f"/x{i}.py", 1)
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
assert len(info) == 5
assert all("diags" in r.getMessage() for r in info)
# ---------------------------------------------------------------------------
# Action-required: WARNING once, DEBUG thereafter (or per call for novel events)
# ---------------------------------------------------------------------------
def test_server_unavailable_warns_once_per_binary(caplog_lsp):
for _ in range(20):
eventlog.log_server_unavailable("pyright", "pyright-langserver")
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
assert len(warns) == 1
assert "pyright-langserver" in warns[0].getMessage()
def test_server_unavailable_separate_per_binary(caplog_lsp):
eventlog.log_server_unavailable("pyright", "pyright-langserver")
eventlog.log_server_unavailable("typescript", "typescript-language-server")
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
assert len(warns) == 2
def test_no_server_configured_warns_once(caplog_lsp):
for _ in range(10):
eventlog.log_no_server_configured("pyright")
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
assert len(warns) == 1
def test_timeout_warns_every_call(caplog_lsp):
for _ in range(3):
eventlog.log_timeout("pyright", "/x.py")
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
assert len(warns) == 3
def test_server_error_warns_every_call(caplog_lsp):
for _ in range(3):
eventlog.log_server_error("pyright", "/x.py", RuntimeError("boom"))
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
assert len(warns) == 3
def test_spawn_failed_warns(caplog_lsp):
eventlog.log_spawn_failed("pyright", "/proj", FileNotFoundError("nope"))
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
assert len(warns) == 1
assert "spawn/initialize failed" in warns[0].getMessage()
# ---------------------------------------------------------------------------
# Format: log lines all carry the lsp[<server_id>] prefix for grep
# ---------------------------------------------------------------------------
def test_log_lines_use_lsp_prefix(caplog_lsp):
eventlog.log_clean("pyright", "/x.py")
eventlog.log_active("pyright", "/proj")
eventlog.log_diagnostics("typescript", "/y.ts", 2)
for r in caplog_lsp.records:
assert r.getMessage().startswith("lsp[")
# ---------------------------------------------------------------------------
# Steady-state contract: 1000 clean writes → 1 INFO at most
# ---------------------------------------------------------------------------
def test_thousand_clean_writes_emit_one_info(caplog_lsp):
"""A long session writes lots of files cleanly; agent.log should
show ONE 'active for' INFO and zero other INFO lines."""
eventlog.log_active("pyright", "/proj")
for _ in range(1000):
eventlog.log_clean("pyright", "/proj/x.py")
info_records = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
assert len(info_records) == 1
assert "active for" in info_records[0].getMessage()
# ---------------------------------------------------------------------------
# Path shortening
# ---------------------------------------------------------------------------
def test_short_path_uses_relative_when_inside_cwd(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
sub = tmp_path / "x.py"
sub.write_text("")
out = eventlog._short_path(str(sub))
assert out == "x.py"
def test_short_path_keeps_absolute_when_outside(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path / "a") if (tmp_path / "a").exists() else None
monkeypatch.chdir(tmp_path)
other = "/var/log/foo.txt"
out = eventlog._short_path(other)
# Outside cwd: keeps absolute (no leading "../")
assert out == "/var/log/foo.txt" or not out.startswith("..")
def test_short_path_handles_empty_string():
assert eventlog._short_path("") == ""
@@ -0,0 +1,320 @@
"""Tests for follow-up fixes to the LSP integration (PR after #24168).
Covers:
1. ``typescript-language-server`` install recipe pulls in ``typescript``
alongside the server, so the npm install command targets both.
2. ``hermes lsp status`` surfaces a ``Backend warnings`` section when
bash-language-server is installed but ``shellcheck`` is missing.
3. ``_check_lint`` returns ``skipped`` (not ``error``) when the linter
command exists on PATH but couldn't actually run — e.g. ``npx tsc``
without the typescript SDK installed. This is what unblocks the
LSP semantic tier on TypeScript files when the user doesn't also
have a project-level ``tsc``.
"""
from __future__ import annotations
import io
from contextlib import redirect_stdout
from unittest.mock import MagicMock, patch
import pytest
from agent.lsp.install import INSTALL_RECIPES
# ---------------------------------------------------------------------------
# Fix 1: typescript install recipe carries the typescript SDK
# ---------------------------------------------------------------------------
def test_typescript_recipe_includes_typescript_sdk():
recipe = INSTALL_RECIPES["typescript-language-server"]
extras = recipe.get("extra_pkgs") or []
assert "typescript" in extras, (
"typescript-language-server requires the `typescript` SDK as a "
"sibling install — without it `initialize` fails with "
"'Could not find a valid TypeScript installation'."
)
def test_install_npm_passes_extras_to_npm_command(tmp_path, monkeypatch):
"""Verify the npm subprocess is invoked with both pkg AND extras."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
captured = {}
def fake_run(cmd, **kwargs):
captured["cmd"] = cmd
# Pretend npm succeeded but binary doesn't exist — install code
# will return None, which is fine for this test.
return MagicMock(returncode=0, stderr="")
from agent.lsp import install as install_mod
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
monkeypatch.setattr(install_mod.shutil, "which", lambda c: "/usr/bin/npm" if c == "npm" else None)
install_mod._install_npm("typescript-language-server", "typescript-language-server",
extra_pkgs=["typescript"])
cmd = captured["cmd"]
assert "typescript-language-server" in cmd
assert "typescript" in cmd
# Both must come AFTER the npm flags, in install-target position
install_idx = cmd.index("install")
assert cmd.index("typescript-language-server") > install_idx
assert cmd.index("typescript") > install_idx
def test_install_npm_works_without_extras(tmp_path, monkeypatch):
"""Backwards compat: pyright-style recipes (no extras) still install."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
captured = {}
def fake_run(cmd, **kwargs):
captured["cmd"] = cmd
return MagicMock(returncode=0, stderr="")
from agent.lsp import install as install_mod
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
monkeypatch.setattr(install_mod.shutil, "which", lambda c: "/usr/bin/npm" if c == "npm" else None)
install_mod._install_npm("pyright", "pyright-langserver")
cmd = captured["cmd"]
assert "pyright" in cmd
# Should not blow up when extra_pkgs is omitted/None
install_targets = [c for c in cmd if not c.startswith("-") and c not in {
"install", "--prefix", str(install_mod.hermes_lsp_bin_dir().parent),
"/usr/bin/npm",
}]
assert install_targets == ["pyright"]
def test_existing_binary_finds_windows_wrapper_in_staging(tmp_path, monkeypatch):
"""Installed Windows shims should satisfy later status/probe calls."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import install as install_mod
wrapper = install_mod.hermes_lsp_bin_dir() / "pyright-langserver.cmd"
wrapper.write_text("@echo off\n")
wrapper.chmod(0o755)
monkeypatch.setattr(install_mod, "_is_windows", lambda: True)
monkeypatch.setattr(install_mod.shutil, "which", lambda _name: None)
assert install_mod._existing_binary("pyright-langserver") == str(wrapper)
assert install_mod.detect_status("pyright") == "installed"
def test_install_pip_finds_windows_scripts_launcher(tmp_path, monkeypatch):
"""pip console scripts can land in Scripts/ on native Windows."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import install as install_mod
def fake_run(cmd, **kwargs):
scripts_dir = install_mod.hermes_lsp_bin_dir().parent / "python-packages" / "Scripts"
scripts_dir.mkdir(parents=True, exist_ok=True)
launcher = scripts_dir / "fake-language-server.exe"
launcher.write_text("launcher\n")
launcher.chmod(0o755)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(install_mod, "_is_windows", lambda: True)
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
resolved = install_mod._install_pip("fake-lsp", "fake-language-server")
assert resolved is not None
assert resolved.endswith("fake-language-server.exe")
assert (install_mod.hermes_lsp_bin_dir() / "fake-language-server.exe").exists()
# ---------------------------------------------------------------------------
# Fix 2: ``hermes lsp status`` surfaces shellcheck-missing for bash
# ---------------------------------------------------------------------------
def test_backend_warnings_quiet_when_bash_not_installed(tmp_path, monkeypatch):
"""No bash → no warning."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import cli as lsp_cli
with patch("shutil.which", return_value=None):
notes = lsp_cli._backend_warnings()
assert notes == []
def test_backend_warnings_quiet_when_bash_and_shellcheck_both_present(tmp_path, monkeypatch):
"""Both installed → no warning."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import cli as lsp_cli
def which(name):
return f"/usr/bin/{name}" # both found
with patch("shutil.which", side_effect=which):
notes = lsp_cli._backend_warnings()
assert notes == []
def test_backend_warnings_fires_when_bash_installed_but_shellcheck_missing(tmp_path, monkeypatch):
"""The exact scenario from the bug report."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import cli as lsp_cli
def which(name):
if name == "bash-language-server":
return "/fake/bin/bash-language-server"
return None # shellcheck missing
with patch("shutil.which", side_effect=which):
notes = lsp_cli._backend_warnings()
assert len(notes) == 1
assert "shellcheck" in notes[0].lower()
assert "bash-language-server" in notes[0].lower()
def test_status_output_includes_backend_warnings_section(tmp_path, monkeypatch):
"""End-to-end: status command output includes the warning section."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
# Pretend bash-language-server is installed but shellcheck is missing
def which(name):
if name == "bash-language-server":
return "/fake/bin/bash-language-server"
return None
from agent.lsp import cli as lsp_cli
buf = io.StringIO()
with patch("shutil.which", side_effect=which), redirect_stdout(buf):
lsp_cli._cmd_status(emit_json=False)
output = buf.getvalue()
assert "Backend warnings" in output
assert "shellcheck" in output
# ---------------------------------------------------------------------------
# Fix 3: tier-1 lint treats unusable linters as ``skipped``, not ``error``
# ---------------------------------------------------------------------------
def test_npx_tsc_missing_treated_as_skipped():
"""The original bug: ``npx tsc`` errors when tsc isn't installed.
Without this fix, the lint result is ``error``, which means the LSP
semantic tier (gated on ``success or skipped``) is skipped — the user
gets a useless tooling-error message instead of real diagnostics.
"""
from tools.file_operations import _looks_like_linter_unusable
npx_failure_output = (
" \n"
" This is not the tsc command you are looking for \n"
" \n"
"\n"
"To get access to the TypeScript compiler, tsc, from the command line either:\n"
"- Use npm install typescript to first add TypeScript to your project before using npx\n"
)
assert _looks_like_linter_unusable("npx", npx_failure_output) is True
def test_real_lint_error_not_classified_as_unusable():
"""A genuine TypeScript type error must NOT be misclassified."""
from tools.file_operations import _looks_like_linter_unusable
real_error = (
"bad.ts:5:1 - error TS2322: Type 'number' is not assignable to type 'string'.\n"
"5 const x: string = greet(42);\n"
" ~~~~~~~~~~~~~~~\n"
)
assert _looks_like_linter_unusable("npx", real_error) is False
def test_unknown_base_cmd_returns_false():
"""Unfamiliar linters fall through and use the normal error path."""
from tools.file_operations import _looks_like_linter_unusable
assert _looks_like_linter_unusable("eslint", "any output") is False
assert _looks_like_linter_unusable("", "anything") is False
def test_check_lint_returns_skipped_when_npx_tsc_unusable(tmp_path):
"""Integration: _check_lint sees npx exit non-zero with the npx banner
and returns a ``skipped`` LintResult so LSP can still run."""
from tools.environments.local import LocalEnvironment
from tools.file_operations import ShellFileOperations
ts_file = tmp_path / "bad.ts"
ts_file.write_text("const x: string = 42;\n")
env = LocalEnvironment()
fops = ShellFileOperations(env)
# Patch _exec to simulate ``npx tsc`` failing because tsc is missing.
npx_banner = (
" \n"
" This is not the tsc command you are looking for \n"
)
def fake_exec(cmd, **kwargs):
result = MagicMock()
result.exit_code = 1
result.stdout = npx_banner
return result
with patch.object(fops, "_exec", side_effect=fake_exec), \
patch.object(fops, "_has_command", return_value=True):
lint = fops._check_lint(str(ts_file))
assert lint.skipped is True, (
f"expected skipped (so LSP runs); got success={lint.success}, "
f"output={lint.output!r}"
)
assert "not usable" in (lint.message or "")
def test_check_lint_returns_error_for_real_ts_type_errors(tmp_path):
"""Sanity: real TypeScript errors still go through the error path."""
from tools.environments.local import LocalEnvironment
from tools.file_operations import ShellFileOperations
ts_file = tmp_path / "bad.ts"
ts_file.write_text("const x: string = 42;\n")
env = LocalEnvironment()
fops = ShellFileOperations(env)
real_tsc_error = (
"bad.ts:1:7 - error TS2322: Type 'number' is not assignable to type 'string'.\n"
"1 const x: string = 42;\n"
" ~\n"
"Found 1 error.\n"
)
def fake_exec(cmd, **kwargs):
result = MagicMock()
result.exit_code = 1
result.stdout = real_tsc_error
return result
with patch.object(fops, "_exec", side_effect=fake_exec), \
patch.object(fops, "_has_command", return_value=True):
lint = fops._check_lint(str(ts_file))
assert lint.skipped is False
assert lint.success is False
assert "TS2322" in lint.output
if __name__ == "__main__": # pragma: no cover
pytest.main([__file__, "-v"])
+144
View File
@@ -0,0 +1,144 @@
"""Tests for service-singleton lifecycle: atexit handler, idempotent shutdown.
These cover the exit-cleanup behavior added to plug the language-server
process leak — without the atexit hook, ``hermes chat`` exits while
pyright/gopls/etc. are still alive on the host.
"""
from __future__ import annotations
import atexit
from unittest.mock import MagicMock
import pytest
from agent import lsp as lsp_module
@pytest.fixture(autouse=True)
def _reset_singleton():
"""Force a clean module state before each test.
Tests in this file share process-global state (the lazy
singleton + atexit registration flag); reset both before and
after every test so order doesn't matter.
"""
lsp_module._service = None
lsp_module._atexit_registered = False
yield
lsp_module._service = None
lsp_module._atexit_registered = False
def test_get_service_registers_atexit_handler_once(monkeypatch):
"""First call to ``get_service`` must register an atexit handler;
subsequent calls must NOT register another one (Python's ``atexit``
runs every registered callable, so a duplicate would shutdown
twice — harmless but wasteful)."""
fake_svc = MagicMock()
fake_svc.is_active.return_value = True
monkeypatch.setattr(
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
)
registrations = []
def fake_register(fn):
registrations.append(fn)
monkeypatch.setattr(atexit, "register", fake_register)
a = lsp_module.get_service()
b = lsp_module.get_service()
c = lsp_module.get_service()
assert a is fake_svc
assert b is fake_svc
assert c is fake_svc
assert len(registrations) == 1
# The registered callable must be our internal shutdown wrapper.
assert registrations[0] is lsp_module._atexit_shutdown
def test_atexit_shutdown_calls_shutdown_service(monkeypatch):
"""The atexit-registered wrapper invokes ``shutdown_service`` and
swallows any exception — by the time atexit fires, the user has
already seen the response and a noisy traceback would be clutter."""
called = []
monkeypatch.setattr(
lsp_module, "shutdown_service", lambda: called.append("shutdown")
)
lsp_module._atexit_shutdown()
assert called == ["shutdown"]
def test_atexit_shutdown_swallows_exceptions(monkeypatch):
def boom():
raise RuntimeError("server already dead")
monkeypatch.setattr(lsp_module, "shutdown_service", boom)
# Must not raise.
lsp_module._atexit_shutdown()
def test_shutdown_service_idempotent(monkeypatch):
"""Calling shutdown twice must be safe — first call cleans up,
second call no-ops (nothing to shut down)."""
fake_svc = MagicMock()
fake_svc.is_active.return_value = True
fake_svc.shutdown = MagicMock()
monkeypatch.setattr(
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
)
monkeypatch.setattr(atexit, "register", lambda fn: None)
lsp_module.get_service()
lsp_module.shutdown_service()
lsp_module.shutdown_service() # must not raise
assert fake_svc.shutdown.call_count == 1
def test_shutdown_service_no_op_when_never_started():
"""Calling shutdown without ever creating the service is safe."""
lsp_module.shutdown_service() # must not raise
def test_shutdown_service_swallows_exception(monkeypatch):
"""An exception during ``svc.shutdown()`` must not propagate —
the caller (often atexit) has nothing useful to do with it."""
fake_svc = MagicMock()
fake_svc.is_active.return_value = True
fake_svc.shutdown = MagicMock(side_effect=RuntimeError("kill -9 already"))
monkeypatch.setattr(
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
)
monkeypatch.setattr(atexit, "register", lambda fn: None)
lsp_module.get_service()
lsp_module.shutdown_service() # must not raise
def test_get_service_returns_none_for_inactive_service(monkeypatch):
"""A service whose ``is_active()`` returns False is treated as
not running — callers see ``None`` and fall back."""
fake_svc = MagicMock()
fake_svc.is_active.return_value = False
monkeypatch.setattr(
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
)
monkeypatch.setattr(atexit, "register", lambda fn: None)
assert lsp_module.get_service() is None
# Subsequent call returns None too — but the inactive instance is
# cached so we don't re-build it on every check.
assert lsp_module.get_service() is None
def test_get_service_returns_none_when_create_fails(monkeypatch):
"""Service factory returning ``None`` (no config, etc.) propagates."""
monkeypatch.setattr(
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: None)
)
monkeypatch.setattr(atexit, "register", lambda fn: None)
assert lsp_module.get_service() is None
+197
View File
@@ -0,0 +1,197 @@
"""Tests for the LSP protocol framing layer.
The framer is small but load-bearing — Content-Length parsing is the
single most common reason for hand-rolled LSP clients to silently
deadlock. These tests exercise:
- exact wire format of outgoing messages (encode_message)
- partial-read tolerance + EOF handling (read_message)
- envelope helpers (request, response, notification, error)
- message classification
"""
from __future__ import annotations
import asyncio
import json
import pytest
from agent.lsp.protocol import (
ERROR_CONTENT_MODIFIED,
ERROR_METHOD_NOT_FOUND,
LSPProtocolError,
LSPRequestError,
classify_message,
encode_message,
make_error_response,
make_notification,
make_request,
make_response,
read_message,
)
# ---------------------------------------------------------------------------
# encode_message
# ---------------------------------------------------------------------------
def test_encode_message_uses_compact_separators_and_utf8():
msg = {"jsonrpc": "2.0", "id": 1, "method": "x", "params": {"k": "ä"}}
out = encode_message(msg)
# Header is plain ASCII Content-Length CRLF CRLF
header_end = out.index(b"\r\n\r\n") + 4
header = out[:header_end].decode("ascii")
body = out[header_end:]
assert "Content-Length:" in header
declared = int(header.split("Content-Length:")[1].split("\r\n")[0].strip())
# Declared length must equal actual body bytes.
assert declared == len(body)
# Body parses as JSON and round-trips.
parsed = json.loads(body.decode("utf-8"))
assert parsed == msg
# Body uses compact separators (no spaces between kv).
assert b'"id":1' in body
def test_encode_message_handles_unicode_in_strings():
msg = {"jsonrpc": "2.0", "method": "log", "params": {"text": "🚀 ünıcödé"}}
out = encode_message(msg)
header_end = out.index(b"\r\n\r\n") + 4
declared = int(out[: out.index(b"\r\n")].split(b": ")[1])
assert declared == len(out[header_end:])
assert json.loads(out[header_end:].decode("utf-8")) == msg
# ---------------------------------------------------------------------------
# read_message
# ---------------------------------------------------------------------------
async def _stream_from_bytes(data: bytes) -> asyncio.StreamReader:
"""Build an asyncio.StreamReader pre-populated with ``data``."""
reader = asyncio.StreamReader()
reader.feed_data(data)
reader.feed_eof()
return reader
@pytest.mark.asyncio
async def test_read_message_round_trip():
msg = {"jsonrpc": "2.0", "method": "ping"}
reader = await _stream_from_bytes(encode_message(msg))
parsed = await read_message(reader)
assert parsed == msg
@pytest.mark.asyncio
async def test_read_message_clean_eof_returns_none():
reader = await _stream_from_bytes(b"")
assert await read_message(reader) is None
@pytest.mark.asyncio
async def test_read_message_truncated_body_raises():
msg = encode_message({"jsonrpc": "2.0", "method": "x"})
truncated = msg[: -3] # cut the body
reader = await _stream_from_bytes(truncated)
with pytest.raises(LSPProtocolError):
await read_message(reader)
@pytest.mark.asyncio
async def test_read_message_missing_content_length_raises():
bad = b"X-Other: 5\r\n\r\n12345"
reader = await _stream_from_bytes(bad)
with pytest.raises(LSPProtocolError):
await read_message(reader)
@pytest.mark.asyncio
async def test_read_message_two_messages_back_to_back():
a = encode_message({"jsonrpc": "2.0", "method": "a"})
b = encode_message({"jsonrpc": "2.0", "method": "b"})
reader = await _stream_from_bytes(a + b)
assert (await read_message(reader))["method"] == "a"
assert (await read_message(reader))["method"] == "b"
@pytest.mark.asyncio
async def test_read_message_rejects_runaway_header():
"""A pathological server that streams headers without ever emitting
the CRLF-CRLF terminator must not loop forever — the 8 KiB cap kicks
in and surfaces a protocol error."""
flood = (b"X-Junk: " + b"A" * 200 + b"\r\n") * 60 # ~12 KiB worth
reader = await _stream_from_bytes(flood)
with pytest.raises(LSPProtocolError) as exc:
await read_message(reader)
assert "8 KiB" in str(exc.value)
# ---------------------------------------------------------------------------
# envelope helpers
# ---------------------------------------------------------------------------
def test_make_request_includes_id_and_method():
msg = make_request(7, "ping", {"v": 1})
assert msg == {"jsonrpc": "2.0", "id": 7, "method": "ping", "params": {"v": 1}}
def test_make_request_omits_params_when_none():
msg = make_request(7, "ping", None)
assert "params" not in msg
def test_make_notification_omits_id():
msg = make_notification("log", {"line": "hi"})
assert "id" not in msg
assert msg["method"] == "log"
def test_make_response_carries_result():
msg = make_response(7, {"ok": True})
assert msg["id"] == 7 and msg["result"] == {"ok": True}
def test_make_error_response_shape():
msg = make_error_response(7, ERROR_CONTENT_MODIFIED, "stale", {"hint": "retry"})
assert msg["error"]["code"] == ERROR_CONTENT_MODIFIED
assert msg["error"]["message"] == "stale"
assert msg["error"]["data"] == {"hint": "retry"}
# ---------------------------------------------------------------------------
# classify_message
# ---------------------------------------------------------------------------
def test_classify_message_request():
msg = {"jsonrpc": "2.0", "id": 1, "method": "x"}
assert classify_message(msg) == ("request", 1)
def test_classify_message_response():
msg = {"jsonrpc": "2.0", "id": 1, "result": None}
assert classify_message(msg) == ("response", 1)
def test_classify_message_notification():
msg = {"jsonrpc": "2.0", "method": "log"}
assert classify_message(msg) == ("notification", "log")
def test_classify_message_invalid():
assert classify_message({"id": 1})[0] == "invalid"
assert classify_message({"jsonrpc": "1.0", "method": "x"})[0] == "invalid"
# ---------------------------------------------------------------------------
# LSPRequestError
# ---------------------------------------------------------------------------
def test_lsp_request_error_carries_code_and_data():
e = LSPRequestError(ERROR_METHOD_NOT_FOUND, "no", {"x": 1})
assert e.code == ERROR_METHOD_NOT_FOUND
assert e.message == "no"
assert e.data == {"x": 1}
+93
View File
@@ -0,0 +1,93 @@
"""Tests for the diagnostic reporter (formatting layer)."""
from __future__ import annotations
from agent.lsp.reporter import (
MAX_PER_FILE,
format_diagnostic,
report_for_file,
truncate,
)
def _diag(line=0, col=0, sev=1, code="E001", source="ls", msg="oops"):
return {
"range": {
"start": {"line": line, "character": col},
"end": {"line": line, "character": col + 1},
},
"severity": sev,
"code": code,
"source": source,
"message": msg,
}
def test_format_diagnostic_uses_one_indexed_position():
line = format_diagnostic(_diag(line=4, col=2))
assert "[5:3]" in line # +1 on both
def test_format_diagnostic_includes_severity_label():
assert format_diagnostic(_diag(sev=1)).startswith("ERROR")
assert format_diagnostic(_diag(sev=2)).startswith("WARN")
assert format_diagnostic(_diag(sev=3)).startswith("INFO")
assert format_diagnostic(_diag(sev=4)).startswith("HINT")
def test_format_diagnostic_includes_code_and_source():
line = format_diagnostic(_diag(code="X42", source="src"))
assert "[X42]" in line
assert "(src)" in line
def test_format_diagnostic_omits_missing_optional_fields():
line = format_diagnostic(
{
"range": {
"start": {"line": 0, "character": 0},
"end": {"line": 0, "character": 0},
},
"severity": 1,
"message": "bare",
}
)
assert "[" not in line.split("]", 1)[1] # no extra brackets after the position
assert "(" not in line
def test_report_for_file_returns_empty_when_only_warnings():
"""Default severity filter is ERROR-only."""
report = report_for_file("/x.py", [_diag(sev=2)])
assert report == ""
def test_report_for_file_emits_block_with_errors():
diag = _diag(msg="real error")
report = report_for_file("/x.py", [diag])
assert "<diagnostics file=\"/x.py\">" in report
assert "real error" in report
assert "</diagnostics>" in report
def test_report_for_file_caps_at_max_per_file():
diags = [_diag(line=i) for i in range(MAX_PER_FILE + 5)]
report = report_for_file("/x.py", diags)
assert "and 5 more" in report
def test_report_for_file_respects_custom_severities():
diag = _diag(sev=2, msg="warn")
report = report_for_file("/x.py", [diag], severities=frozenset({1, 2}))
assert "warn" in report
def test_truncate_below_limit_unchanged():
s = "abc" * 100
assert truncate(s, limit=4000) == s
def test_truncate_above_limit_appends_marker():
s = "x" * 10000
out = truncate(s, limit=200)
assert out.endswith("[truncated]")
assert len(out) <= 200
+176
View File
@@ -0,0 +1,176 @@
"""Tests for the synchronous LSPService wrapper.
Drives the service through ``snapshot_baseline`` →
``get_diagnostics_sync`` against the mock LSP server, exercising the
delta filter that ``tools/file_operations._check_lint_delta`` relies
on.
"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
from agent.lsp.manager import LSPService
from agent.lsp.servers import (
SERVERS,
ServerContext,
ServerDef,
SpawnSpec,
)
MOCK_SERVER = str(Path(__file__).parent / "_mock_lsp_server.py")
def _install_mock_server(monkeypatch, script: str = "errors", server_id: str = "pyright"):
"""Replace one registered server with a wrapper that spawns the mock.
We reuse ``pyright`` so .py files route to it. This keeps the
test free of any LSP toolchain dependency.
"""
target_index = next(i for i, s in enumerate(SERVERS) if s.server_id == server_id)
original = SERVERS[target_index]
def _spawn(root: str, ctx: ServerContext) -> SpawnSpec:
env = {"MOCK_LSP_SCRIPT": script}
return SpawnSpec(
command=[sys.executable, MOCK_SERVER],
workspace_root=root,
cwd=root,
env=env,
initialization_options={},
)
replacement = ServerDef(
server_id=server_id,
extensions=original.extensions,
resolve_root=lambda fp, ws: ws, # always use workspace root
build_spawn=_spawn,
seed_first_push=False,
description="mock " + server_id,
)
# Patch the SERVERS list element directly + restore on teardown.
SERVERS[target_index] = replacement
yield
SERVERS[target_index] = original
@pytest.fixture
def mock_pyright(monkeypatch, tmp_path):
"""Install the mock as ``pyright`` and create a fake git workspace."""
repo = tmp_path / "repo"
repo.mkdir()
(repo / ".git").mkdir()
(repo / "pyproject.toml").write_text("") # so pyright's root resolver finds it
monkeypatch.chdir(str(repo))
gen = _install_mock_server(monkeypatch, "errors", "pyright")
next(gen)
yield repo
try:
next(gen)
except StopIteration:
pass
def test_service_returns_empty_when_disabled(tmp_path):
svc = LSPService(
enabled=False,
wait_mode="document",
wait_timeout=2.0,
install_strategy="auto",
)
assert not svc.is_active()
f = tmp_path / "x.py"
f.write_text("")
assert svc.get_diagnostics_sync(str(f)) == []
svc.shutdown()
def test_service_skips_files_outside_workspace(tmp_path):
"""Files outside any git worktree must not trigger LSP."""
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=2.0,
install_strategy="manual",
)
f = tmp_path / "x.py"
f.write_text("")
# No .git anywhere — service should report not enabled for this file.
assert not svc.enabled_for(str(f))
svc.shutdown()
def test_service_e2e_delta_filter(mock_pyright):
"""End-to-end: snapshot baseline → wait → delta returned."""
repo = mock_pyright
f = repo / "x.py"
f.write_text("print('hi')\n")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=3.0,
install_strategy="manual",
)
try:
assert svc.enabled_for(str(f))
# Baseline first — server pushes 1 error.
svc.snapshot_baseline(str(f))
# Re-poll: same error is in baseline, so delta is empty.
new_diags = svc.get_diagnostics_sync(str(f))
assert new_diags == []
finally:
svc.shutdown()
def test_service_e2e_delta_filter_with_line_shift(mock_pyright):
"""End-to-end: an edit that shifts the diagnostic's line still
filters correctly when ``line_shift`` is supplied.
The mock LSP server emits a fixed error at line 0; for this test
we don't need to actually shift the server's output — we just
need to prove that supplying a line_shift through the API works
and doesn't break the existing delta path. The unit tests in
test_delta_key.py cover the shift semantics in detail.
"""
repo = mock_pyright
f = repo / "x.py"
f.write_text("print('hi')\n")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=3.0,
install_strategy="manual",
)
try:
svc.snapshot_baseline(str(f))
# Identity shift — should behave exactly like no shift.
new_diags = svc.get_diagnostics_sync(str(f), line_shift=lambda L: L)
assert new_diags == []
finally:
svc.shutdown()
def test_service_status_includes_clients(mock_pyright):
repo = mock_pyright
f = repo / "x.py"
f.write_text("")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=3.0,
install_strategy="manual",
)
try:
svc.get_diagnostics_sync(str(f))
info = svc.get_status()
assert info["enabled"] is True
assert any(c["server_id"] == "pyright" for c in info["clients"])
finally:
svc.shutdown()
@@ -0,0 +1,210 @@
"""Skip the per-file shell linter when LSP will handle the same file.
The per-file ``npx tsc --noEmit FILE.ts`` shell linter cannot see
``tsconfig.json`` (a documented ``tsc`` quirk: explicit file args bypass
the project config), so it defaults to no-lib / ES5 and floods the
agent's lint field with phantom "Cannot find 'Promise' / 'Map' / 'Set' /
'ReadonlySet' / 'Iterable' / 'imul' / …" errors on every edit — up to
25K tokens per patch. The LSP tier (``tsserver`` via
typescript-language-server) reads tsconfig correctly and surfaces real
diagnostics in the ``lsp_diagnostics`` field of the WriteResult /
PatchResult.
These tests pin the contract:
- When LSP is active AND ``enabled_for(path)`` for a ``.ts`` / ``.go``
/ ``.rs`` file, ``_check_lint`` returns ``skipped`` without invoking
the shell linter at all.
- When LSP is inactive or disabled-for-path, the shell linter runs
exactly as before (regression guard for the default config).
- The skip only applies to extensions in
``_SHELL_LINTER_LSP_REDUNDANT`` — Python ``py_compile`` and
``node --check`` keep running unconditionally because they're fast,
file-local, and correct.
- ``.tsx`` is intentionally NOT in either ``LINTERS`` or
``_SHELL_LINTER_LSP_REDUNDANT``: it had no ``LINTERS`` entry
pre-PR (so it was already implicitly ``skipped`` via the
``ext not in LINTERS`` branch) and adding one would have inherited
``.ts``'s broken ``tsc --noEmit FILE`` invocation for LSP-disabled
users. When LSP IS enabled, ``.tsx`` is still covered by
typescript-language-server via ``_maybe_lsp_diagnostics`` — the
diagnostics show up on ``lsp_diagnostics``, not ``lint``.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
def _make_fops():
from tools.environments.local import LocalEnvironment
from tools.file_operations import ShellFileOperations
return ShellFileOperations(LocalEnvironment())
@pytest.mark.parametrize("ext", [".ts", ".go", ".rs"])
def test_shell_linter_skipped_when_lsp_will_handle(ext, tmp_path):
"""When LSP is active and enabled_for(path), shell linter is skipped.
The shell linter's _exec must NOT be called — that's the whole
point. We assert by patching ``_exec`` to raise, so any accidental
invocation surfaces as a test failure.
"""
fops = _make_fops()
src = tmp_path / f"bad{ext}"
src.write_text("intentionally invalid content\n")
def _exec_must_not_run(*args, **kwargs): # pragma: no cover
raise AssertionError(
"shell linter was invoked despite LSP claiming the file"
)
with patch.object(fops, "_lsp_will_handle", return_value=True), \
patch.object(fops, "_exec", side_effect=_exec_must_not_run), \
patch.object(fops, "_has_command", return_value=True):
result = fops._check_lint(str(src))
assert result.skipped is True
assert "LSP" in (result.message or "")
@pytest.mark.parametrize("ext", [".ts", ".go", ".rs"])
def test_shell_linter_runs_when_lsp_inactive(ext, tmp_path):
"""When LSP is inactive (default config, no service, remote backend, ...),
the shell linter runs as before — no behavior change."""
fops = _make_fops()
src = tmp_path / f"clean{ext}"
src.write_text("// content\n")
fake_result = MagicMock()
fake_result.exit_code = 0
fake_result.stdout = ""
with patch.object(fops, "_lsp_will_handle", return_value=False), \
patch.object(fops, "_exec", return_value=fake_result) as exec_mock, \
patch.object(fops, "_has_command", return_value=True):
result = fops._check_lint(str(src))
# _exec must have been called — proving the shell linter ran.
assert exec_mock.called, "shell linter did NOT run when LSP was inactive"
assert result.success is True
@pytest.mark.parametrize("ext", [".py", ".js"])
def test_lsp_does_not_skip_non_redundant_extensions(ext, tmp_path):
"""``py_compile`` and ``node --check`` keep running even when an LSP
server (pyright/pylsp/typescript-language-server-for-JS) is active —
they're fast, file-local, and correct, so there's no upside to
suppressing them.
"""
fops = _make_fops()
src = tmp_path / f"clean{ext}"
src.write_text("# valid\n" if ext == ".py" else "// valid\n")
fake_result = MagicMock()
fake_result.exit_code = 0
fake_result.stdout = ""
# Even with LSP claiming the file, the shell linter must still run
# for these extensions.
with patch.object(fops, "_lsp_will_handle", return_value=True), \
patch.object(fops, "_exec", return_value=fake_result) as exec_mock, \
patch.object(fops, "_has_command", return_value=True):
fops._check_lint(str(src))
assert exec_mock.called, (
f"shell linter for {ext} did not run despite being in the "
"'always-run' set (py_compile / node --check)"
)
def test_lsp_will_handle_returns_false_when_service_is_none(tmp_path):
"""``_lsp_will_handle`` must return False when the LSP service hasn't
been initialized — otherwise we'd accidentally skip the shell linter
on systems where LSP isn't configured at all."""
fops = _make_fops()
src = tmp_path / "foo.ts"
src.write_text("const x = 1\n")
with patch.object(fops, "_lsp_local_only", return_value=True), \
patch("agent.lsp.get_service", return_value=None):
assert fops._lsp_will_handle(str(src)) is False
def test_lsp_will_handle_returns_false_on_remote_backend(tmp_path):
"""LSP servers run on the host process — remote backends (Docker,
SSH, Modal, …) keep files inside the sandbox where the host LSP
can't reach them. ``_lsp_will_handle`` must short-circuit before
calling into the service in that case."""
fops = _make_fops()
src = tmp_path / "foo.ts"
src.write_text("const x = 1\n")
with patch.object(fops, "_lsp_local_only", return_value=False), \
patch("agent.lsp.get_service") as get_service_mock:
result = fops._lsp_will_handle(str(src))
assert result is False
# Importantly: we never even consulted the service.
assert not get_service_mock.called
def test_lsp_will_handle_swallows_enabled_for_exception(tmp_path):
"""A flaky LSP service must never break the shell-linter fallback —
if ``enabled_for`` raises, we treat the file as "not handled" so the
shell linter still runs."""
fops = _make_fops()
src = tmp_path / "foo.ts"
src.write_text("const x = 1\n")
fake_svc = MagicMock()
fake_svc.enabled_for.side_effect = RuntimeError("server crashed")
with patch.object(fops, "_lsp_local_only", return_value=True), \
patch("agent.lsp.get_service", return_value=fake_svc):
assert fops._lsp_will_handle(str(src)) is False
def test_tsx_stays_out_of_linters_table_for_default_compatibility():
"""Regression: keep ``.tsx`` out of ``LINTERS`` so users with LSP
DISABLED don't suddenly get the broken ``npx tsc --noEmit FILE.tsx``
invocation that ``.ts`` historically used to get.
Pre-PR behavior: ``.tsx`` had no entry in ``LINTERS``, so it fell
through to ``ext not in LINTERS`` → ``LintResult(skipped=True,
message="No linter for .tsx files")``. This PR preserves that for
the default config.
When LSP IS enabled, ``.tsx`` is still covered by the LSP tier via
``_maybe_lsp_diagnostics`` (typescript-language-server claims
``.tsx`` in its extensions list) — the diagnostics show up in the
``lsp_diagnostics`` field, not the ``lint`` field.
"""
from tools.file_operations import LINTERS, _SHELL_LINTER_LSP_REDUNDANT
assert ".tsx" not in LINTERS
assert ".tsx" not in _SHELL_LINTER_LSP_REDUNDANT
def test_tsx_default_check_lint_returns_skipped(tmp_path):
"""End-to-end: ``.tsx`` files get ``LintResult(skipped=True)`` from
``_check_lint`` regardless of LSP status — this is the no-regression
contract that addresses Copilot review #3271017282."""
fops = _make_fops()
src = tmp_path / "foo.tsx"
src.write_text("export const X = () => <div/>\n")
# Even with LSP claiming the file, no shell linter runs for .tsx
# because there's no LINTERS entry — the ``ext not in LINTERS``
# branch fires before the LSP short-circuit is consulted.
with patch.object(fops, "_lsp_will_handle", return_value=True), \
patch.object(fops, "_exec") as exec_mock:
result = fops._check_lint(str(src))
assert result.skipped is True
assert not exec_mock.called, "no shell linter should run for .tsx"
if __name__ == "__main__": # pragma: no cover
pytest.main([__file__, "-v"])
+139
View File
@@ -0,0 +1,139 @@
"""Tests for workspace + project-root resolution."""
from __future__ import annotations
import os
from pathlib import Path
import pytest
from agent.lsp.workspace import (
clear_cache,
find_git_worktree,
is_inside_workspace,
nearest_root,
normalize_path,
resolve_workspace_for_file,
)
@pytest.fixture(autouse=True)
def _clear():
clear_cache()
yield
clear_cache()
def test_find_git_worktree_returns_none_outside_repo(tmp_path: Path):
sub = tmp_path / "sub"
sub.mkdir()
assert find_git_worktree(str(sub)) is None
def test_find_git_worktree_finds_dotgit(tmp_path: Path):
repo = tmp_path / "repo"
repo.mkdir()
(repo / ".git").mkdir()
sub = repo / "src" / "deep"
sub.mkdir(parents=True)
assert find_git_worktree(str(sub)) == str(repo)
def test_find_git_worktree_handles_dotgit_file(tmp_path: Path):
"""``.git`` can also be a file (gitfile pointing into a worktree)."""
repo = tmp_path / "repo"
repo.mkdir()
(repo / ".git").write_text("gitdir: /elsewhere\n")
assert find_git_worktree(str(repo)) == str(repo)
def test_is_inside_workspace_true_for_subpath(tmp_path: Path):
root = tmp_path / "p"
root.mkdir()
sub = root / "x" / "y.py"
sub.parent.mkdir(parents=True)
sub.write_text("")
assert is_inside_workspace(str(sub), str(root))
def test_is_inside_workspace_false_for_unrelated(tmp_path: Path):
a = tmp_path / "a"
b = tmp_path / "b"
a.mkdir()
b.mkdir()
f = b / "x.py"
f.write_text("")
assert not is_inside_workspace(str(f), str(a))
def test_nearest_root_finds_first_marker(tmp_path: Path):
root = tmp_path / "p"
deep = root / "src" / "pkg"
deep.mkdir(parents=True)
(root / "pyproject.toml").write_text("")
found = nearest_root(str(deep / "mod.py"), ["pyproject.toml"])
assert found == str(root)
def test_nearest_root_excludes_take_priority(tmp_path: Path):
"""If an exclude marker matches first, return None."""
root = tmp_path / "p"
sub = root / "deno-app"
sub.mkdir(parents=True)
(sub / "deno.json").write_text("{}")
(root / "package.json").write_text("{}") # would match if not for exclude
found = nearest_root(
str(sub / "main.ts"),
["package.json"],
excludes=["deno.json"],
)
assert found is None
def test_nearest_root_returns_none_when_no_marker(tmp_path: Path):
f = tmp_path / "x.py"
f.write_text("")
assert nearest_root(str(f), ["pyproject.toml"]) is None
def test_resolve_workspace_for_file_uses_cwd_first(tmp_path: Path, monkeypatch):
repo = tmp_path / "repo"
(repo / ".git").mkdir(parents=True)
file_path = repo / "x.py"
file_path.write_text("")
# cwd is inside the repo
monkeypatch.chdir(str(repo))
root, gated = resolve_workspace_for_file(str(file_path))
assert root == str(repo)
assert gated is True
def test_resolve_workspace_for_file_no_repo_returns_none(tmp_path: Path, monkeypatch):
monkeypatch.chdir(str(tmp_path))
f = tmp_path / "x.py"
f.write_text("")
root, gated = resolve_workspace_for_file(str(f))
assert root is None
assert gated is False
def test_resolve_workspace_falls_back_to_file_location(tmp_path: Path, monkeypatch):
"""When cwd isn't a git repo but the file is inside one, we still
discover the workspace from the file's path."""
not_a_repo = tmp_path / "loose"
not_a_repo.mkdir()
monkeypatch.chdir(str(not_a_repo))
repo = tmp_path / "actual-repo"
(repo / ".git").mkdir(parents=True)
f = repo / "x.py"
f.write_text("")
root, gated = resolve_workspace_for_file(str(f))
assert root == str(repo)
assert gated is True
def test_normalize_path_expands_tilde(monkeypatch):
monkeypatch.setenv("HOME", "/home/user")
p = normalize_path("~/x.py")
assert p == os.path.abspath("/home/user/x.py")
File diff suppressed because it is too large Load Diff
+163
View File
@@ -0,0 +1,163 @@
"""Tests for Bug #12905 fixes in agent/anthropic_adapter.py — macOS Keychain support."""
import json
from unittest.mock import patch, MagicMock
from agent.anthropic_adapter import (
_read_claude_code_credentials_from_keychain,
read_claude_code_credentials,
)
class TestReadClaudeCodeCredentialsFromKeychain:
"""Bug 4: macOS Keychain support for Claude Code >=2.1.114."""
def test_returns_none_on_linux(self):
"""Keychain reading is Darwin-only; must return None on other platforms."""
with patch("agent.anthropic_adapter.platform.system", return_value="Linux"):
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_on_windows(self):
with patch("agent.anthropic_adapter.platform.system", return_value="Windows"):
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_when_security_command_not_found(self):
"""OSError from missing security binary must be handled gracefully."""
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run",
side_effect=OSError("security not found")):
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_on_nonzero_exit_code(self):
"""security returns non-zero when the Keychain entry doesn't exist."""
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_for_empty_stdout(self):
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_for_non_json_payload(self):
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="not valid json", stderr="")
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_when_password_field_is_missing_claude_ai_oauth(self):
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(
returncode=0,
stdout=json.dumps({"someOtherService": {"accessToken": "tok"}}),
stderr="",
)
assert _read_claude_code_credentials_from_keychain() is None
def test_returns_none_when_access_token_is_empty(self):
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(
returncode=0,
stdout=json.dumps({"claudeAiOauth": {"accessToken": "", "refreshToken": "x"}}),
stderr="",
)
assert _read_claude_code_credentials_from_keychain() is None
def test_parses_valid_keychain_entry(self):
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(
returncode=0,
stdout=json.dumps({
"claudeAiOauth": {
"accessToken": "kc-access-token-abc",
"refreshToken": "kc-refresh-token-xyz",
"expiresAt": 9999999999999,
}
}),
stderr="",
)
creds = _read_claude_code_credentials_from_keychain()
assert creds is not None
assert creds["accessToken"] == "kc-access-token-abc"
assert creds["refreshToken"] == "kc-refresh-token-xyz"
assert creds["expiresAt"] == 9999999999999
assert creds["source"] == "macos_keychain"
class TestReadClaudeCodeCredentialsPriority:
"""Bug 4: Keychain must be checked before the JSON file."""
def test_keychain_takes_priority_over_json_file(self, tmp_path, monkeypatch):
"""When both Keychain and JSON file have credentials, Keychain wins."""
# Set up JSON file with "older" token
json_cred_file = tmp_path / ".claude" / ".credentials.json"
json_cred_file.parent.mkdir(parents=True)
json_cred_file.write_text(json.dumps({
"claudeAiOauth": {
"accessToken": "json-token",
"refreshToken": "json-refresh",
"expiresAt": 9999999999999,
}
}))
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
# Mock Keychain to return a "newer" token
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(
returncode=0,
stdout=json.dumps({
"claudeAiOauth": {
"accessToken": "keychain-token",
"refreshToken": "keychain-refresh",
"expiresAt": 9999999999999,
}
}),
stderr="",
)
creds = read_claude_code_credentials()
# Keychain token should be returned, not JSON file token
assert creds is not None
assert creds["accessToken"] == "keychain-token"
assert creds["source"] == "macos_keychain"
def test_falls_back_to_json_when_keychain_returns_none(self, tmp_path, monkeypatch):
"""When Keychain has no entry, JSON file is used as fallback."""
json_cred_file = tmp_path / ".claude" / ".credentials.json"
json_cred_file.parent.mkdir(parents=True)
json_cred_file.write_text(json.dumps({
"claudeAiOauth": {
"accessToken": "json-fallback-token",
"refreshToken": "json-refresh",
"expiresAt": 9999999999999,
}
}))
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
# Simulate Keychain entry not found
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
creds = read_claude_code_credentials()
assert creds is not None
assert creds["accessToken"] == "json-fallback-token"
assert creds["source"] == "claude_code_credentials_file"
def test_returns_none_when_neither_keychain_nor_json_has_creds(self, tmp_path, monkeypatch):
"""No credentials anywhere — must return None cleanly."""
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
creds = read_claude_code_credentials()
assert creds is None
@@ -0,0 +1,248 @@
"""Tests for GH-25255: Anthropic OAuth mcp_ prefix stripping.
When strip_tool_prefix=True (Anthropic OAuth path), the transport must only
strip the ``mcp_`` prefix from OAuth-injected tools, NOT from Hermes-native
MCP server tools that are registered under their full ``mcp_<server>_<tool>``
name in the tool registry.
"""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import patch
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_tool_use_block(name: str, block_id: str = "tc_1", input_data: dict | None = None):
"""Create a fake Anthropic tool_use content block."""
return SimpleNamespace(
type="tool_use",
id=block_id,
name=name,
input=input_data or {"query": "test"},
)
def _make_response(*blocks, stop_reason="end_turn"):
"""Create a fake Anthropic Messages response."""
return SimpleNamespace(
content=list(blocks),
stop_reason=stop_reason,
model="claude-sonnet-4",
usage=SimpleNamespace(input_tokens=100, output_tokens=50),
)
class _FakeRegistry:
"""Minimal fake tool registry for testing prefix stripping logic."""
def __init__(self, registered_names: set[str]):
self._names = registered_names
def get_entry(self, name: str):
if name in self._names:
return SimpleNamespace(name=name) # truthy = tool exists
return None
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestAnthropicMcpPrefixStrip:
"""Verify that strip_tool_prefix only strips OAuth-injected prefixes."""
def _get_transport(self):
from agent.transports.anthropic import AnthropicTransport
return AnthropicTransport()
def test_strips_prefix_for_oauth_injected_tool(self):
"""OAuth tools: mcp_read_file -> read_file (stripped).
The tool was registered as 'read_file' in the registry.
Anthropic sees 'mcp_read_file' because Hermes adds the prefix.
On response, we must strip it back to 'read_file'.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_read_file")
response = _make_response(block)
registry = _FakeRegistry({"read_file", "terminal", "web_search"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "read_file"
def test_preserves_native_mcp_server_tool_name(self):
"""Native MCP tools: mcp_composio_SEARCH -> mcp_composio_SEARCH (kept).
The tool is registered with the full mcp_ prefix in the registry.
Stripping would break registry lookup.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_composio_COMPOSIO_SEARCH_TOOLS")
response = _make_response(block)
registry = _FakeRegistry({
"mcp_composio_COMPOSIO_SEARCH_TOOLS",
"mcp_composio_COMPOSIO_GET_TOOL_SCHEMAS",
"read_file",
})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "mcp_composio_COMPOSIO_SEARCH_TOOLS"
def test_no_strip_when_flag_false(self):
"""When strip_tool_prefix=False, names are never modified."""
transport = self._get_transport()
block = _make_tool_use_block("mcp_read_file")
response = _make_response(block)
registry = _FakeRegistry({"read_file"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=False)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "mcp_read_file"
def test_no_strip_when_not_mcp_prefixed(self):
"""Non-mcp_ names are untouched regardless of strip flag."""
transport = self._get_transport()
block = _make_tool_use_block("web_search")
response = _make_response(block)
registry = _FakeRegistry({"web_search"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "web_search"
def test_preserves_name_when_neither_in_registry(self):
"""When neither stripped nor full name is in registry, keep full name.
Safety fallback: if we can't determine the type, prefer the full name
since it's what the LLM was told about.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_unknown_tool")
response = _make_response(block)
registry = _FakeRegistry({"read_file"}) # neither name registered
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "mcp_unknown_tool"
def test_mixed_tools_same_response(self):
"""Both OAuth and native MCP tools in the same response."""
transport = self._get_transport()
block1 = _make_tool_use_block("mcp_read_file", block_id="tc_1")
block2 = _make_tool_use_block("mcp_composio_SEARCH", block_id="tc_2")
block3 = _make_tool_use_block("mcp_composio_SEARCH", block_id="tc_3") # also registered natively
response = _make_response(block1, block2, block3)
registry = _FakeRegistry({
"read_file", # OAuth-injected
"mcp_composio_SEARCH", # native MCP
})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 3
# OAuth tool: stripped
assert result.tool_calls[0].name == "read_file"
# Native MCP: preserved (both stripped and full are registered, full wins)
assert result.tool_calls[1].name == "mcp_composio_SEARCH"
assert result.tool_calls[2].name == "mcp_composio_SEARCH"
def test_both_stripped_and_full_registered_prefers_full(self):
"""Edge case: both 'foo' and 'mcp_foo' exist in registry.
Keep 'mcp_foo' (the original name) since it's what the LLM requested.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_foo")
response = _make_response(block)
registry = _FakeRegistry({"foo", "mcp_foo"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
# Both exist — the condition `get_entry(stripped) and not get_entry(name)`
# is False because get_entry(name) IS truthy, so we keep the full name.
assert result.tool_calls[0].name == "mcp_foo"
class TestAnthropicOAuthOutgoingPrefix:
"""Verify the outgoing-side companion fix: build_anthropic_kwargs must not
double-prefix tool names that already start with ``mcp_`` (native MCP server
tools registered as ``mcp_<server>_<tool>``). GH-25255."""
def _build(self, tools, is_oauth=True):
from agent.anthropic_adapter import build_anthropic_kwargs
return build_anthropic_kwargs(
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "Hi"}],
tools=tools,
max_tokens=4096,
reasoning_config=None,
is_oauth=is_oauth,
)
def test_oauth_adds_prefix_to_bare_tool_name(self):
"""OAuth + bare name → prefix added (existing Claude Code convention)."""
kwargs = self._build([{
"type": "function",
"function": {"name": "read_file", "description": "x", "parameters": {}},
}])
names = [t["name"] for t in kwargs["tools"]]
assert names == ["mcp_read_file"]
def test_oauth_does_not_double_prefix_native_mcp_tool(self):
"""OAuth + already-prefixed native MCP name → left alone."""
kwargs = self._build([{
"type": "function",
"function": {
"name": "mcp_composio_COMPOSIO_SEARCH_TOOLS",
"description": "x",
"parameters": {},
},
}])
names = [t["name"] for t in kwargs["tools"]]
# Must NOT become "mcp_mcp_composio_..." — that breaks the round-trip
# because normalize_response only strips ONE mcp_ prefix.
assert names == ["mcp_composio_COMPOSIO_SEARCH_TOOLS"]
def test_oauth_mixed_native_and_bare_tools(self):
"""Mixed: native MCP preserved, bare names prefixed."""
kwargs = self._build([
{"type": "function", "function": {"name": "read_file",
"description": "x", "parameters": {}}},
{"type": "function", "function": {"name": "mcp_composio_SEARCH",
"description": "y", "parameters": {}}},
{"type": "function", "function": {"name": "terminal",
"description": "z", "parameters": {}}},
])
names = sorted(t["name"] for t in kwargs["tools"])
assert names == ["mcp_composio_SEARCH", "mcp_read_file", "mcp_terminal"]
def test_non_oauth_path_untouched(self):
"""Non-OAuth requests never get the prefix — schemas pass through as-is."""
kwargs = self._build([
{"type": "function", "function": {"name": "read_file",
"description": "x", "parameters": {}}},
{"type": "function", "function": {"name": "mcp_composio_SEARCH",
"description": "y", "parameters": {}}},
], is_oauth=False)
names = sorted(t["name"] for t in kwargs["tools"])
assert names == ["mcp_composio_SEARCH", "read_file"]
+176
View File
@@ -0,0 +1,176 @@
"""Regression tests for the Anthropic OAuth PKCE flow.
Guards against re-introducing the bug where the PKCE ``code_verifier`` was
reused as the OAuth ``state`` parameter, leaking the verifier via the
authorization URL (browser history, Referer headers, auth-server logs) and
removing CSRF protection on the callback path.
History:
- PR #1775 first fixed this on ``run_hermes_oauth_login()``.
- PR #2647 (b17e5c10) added ``run_hermes_oauth_login_pure()`` and silently
copy-pasted the pre-#1775 vulnerable pattern.
- PR #3107 removed the old function, leaving only the regressed copy.
- PR #10699 (issue #10693) fixed the regression on the surviving function.
"""
from __future__ import annotations
import json
from typing import Any, Dict
from urllib.parse import parse_qs, urlparse
def _patch_oauth_flow(
monkeypatch,
*,
callback_code: str,
token_response: Dict[str, Any] | None = None,
capture_token_request: Dict[str, Any] | None = None,
capture_auth_url: Dict[str, str] | None = None,
) -> None:
"""Wire up monkeypatches that let ``run_hermes_oauth_login_pure()`` run
end-to-end without touching a real browser, stdin, or HTTP endpoint.
``callback_code`` is the literal string the user would paste back into the
terminal (``"<code>#<state>"`` format).
``capture_token_request`` and ``capture_auth_url`` are out-dict captures
so the test can introspect what was sent to the auth URL and the token
endpoint, respectively.
"""
import urllib.request
if token_response is None:
token_response = {
"access_token": "sk-ant-test-access",
"refresh_token": "sk-ant-test-refresh",
"expires_in": 3600,
}
def fake_open(url):
if capture_auth_url is not None:
capture_auth_url["url"] = url
return True
monkeypatch.setattr("webbrowser.open", fake_open)
# The flow now gates webbrowser.open() behind a graphical-browser check so
# it never launches a console browser (w3m/lynx) inside the terminal. Tests
# run headless, so force the GUI path to True — the URL capture relies on
# webbrowser.open() being invoked.
monkeypatch.setattr(
"hermes_cli.auth._can_open_graphical_browser", lambda: True
)
monkeypatch.setattr("builtins.input", lambda *_a, **_kw: callback_code)
class _FakeResponse:
def __init__(self, body: bytes) -> None:
self._body = body
def __enter__(self):
return self
def __exit__(self, *_exc):
return False
def read(self):
return self._body
def fake_urlopen(req, *_a, **_kw):
if capture_token_request is not None:
capture_token_request["url"] = req.full_url
capture_token_request["data"] = json.loads(req.data.decode())
capture_token_request["headers"] = dict(req.headers)
return _FakeResponse(json.dumps(token_response).encode())
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
def test_authorization_url_state_is_not_pkce_verifier(monkeypatch, tmp_path):
"""The ``state`` parameter in the authorization URL must NOT equal the
PKCE ``code_verifier``.
Reusing the verifier as state leaks the verifier into browser history,
Referer headers, and auth-server access logs — defeating RFC 7636.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
captured_url: Dict[str, str] = {}
captured_token: Dict[str, Any] = {}
_patch_oauth_flow(
monkeypatch,
# state echoed back unchanged so the CSRF guard passes
callback_code="auth-code-from-anthropic#PLACEHOLDER",
capture_auth_url=captured_url,
capture_token_request=captured_token,
)
# Stub the callback parse: we need the state echoed back to match. To do
# that without hardcoding the state value, override input() AFTER seeing
# the auth URL.
import builtins
real_input_calls = {"count": 0}
def fake_input(*_a, **_kw):
real_input_calls["count"] += 1
# First (and only) call is the "Authorization code:" prompt.
url = captured_url.get("url", "")
qs = parse_qs(urlparse(url).query)
state = qs.get("state", [""])[0]
return f"auth-code-from-anthropic#{state}"
monkeypatch.setattr(builtins, "input", fake_input)
from agent.anthropic_adapter import run_hermes_oauth_login_pure
result = run_hermes_oauth_login_pure()
assert result is not None, "OAuth flow should succeed with matching state"
url = captured_url["url"]
qs = parse_qs(urlparse(url).query)
assert "state" in qs and qs["state"][0], "authorization URL must include state"
assert "code_challenge" in qs, "authorization URL must include code_challenge"
state_in_url = qs["state"][0]
verifier_sent = captured_token["data"]["code_verifier"]
# The whole point: state and verifier must be independent values.
assert state_in_url != verifier_sent, (
"PKCE code_verifier was reused as OAuth state — regression of #10693 / "
"#1775. The verifier is supposed to be a secret known only to the "
"client; placing it in the authorization URL leaks it via browser "
"history, Referer headers, and auth-server logs."
)
# And the verifier MUST NOT appear anywhere in the URL.
assert verifier_sent not in url, (
"PKCE verifier leaked into authorization URL — regression of #10693"
)
def test_callback_state_mismatch_aborts(monkeypatch, tmp_path, caplog):
"""If the state returned in the callback does not match the one we sent
in the authorization URL, the flow must abort before exchanging the code.
Without this check, an attacker who tricks the user into pasting a
crafted ``<code>#<state>`` string can complete the token exchange — the
CSRF protection that ``state`` is supposed to provide (RFC 6749 §10.12)
would be absent.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
captured_token: Dict[str, Any] = {}
_patch_oauth_flow(
monkeypatch,
callback_code="attacker-code#attacker-state-does-not-match",
capture_token_request=captured_token,
)
from agent.anthropic_adapter import run_hermes_oauth_login_pure
result = run_hermes_oauth_login_pure()
assert result is None, "mismatched state must abort the flow"
assert "url" not in captured_token, (
"token exchange must NOT happen when state mismatches"
)
@@ -0,0 +1,76 @@
"""Tests for Arcee Trinity Large Thinking per-model overrides.
Arcee Trinity Large Thinking is a reasoning model that wants:
- Fixed temperature=0.5 (vs the global default)
- Compression threshold=0.75 (delay compression to preserve reasoning context)
The helpers must match the bare model name, including when it arrives via
OpenRouter as ``arcee-ai/trinity-large-thinking``, but must NOT hit sibling
Arcee models like trinity-large-preview or trinity-mini.
"""
from __future__ import annotations
import pytest
from agent.auxiliary_client import (
_compression_threshold_for_model,
_fixed_temperature_for_model,
_is_arcee_trinity_thinking,
)
@pytest.mark.parametrize(
"model",
[
"trinity-large-thinking",
"arcee-ai/trinity-large-thinking",
"Arcee-AI/Trinity-Large-Thinking", # case-insensitive
" trinity-large-thinking ", # whitespace tolerant
],
)
def test_is_arcee_trinity_thinking_matches(model: str) -> None:
assert _is_arcee_trinity_thinking(model) is True
@pytest.mark.parametrize(
"model",
[
None,
"",
"trinity-large-preview",
"arcee-ai/trinity-large-preview:free",
"trinity-mini",
"arcee-ai/trinity-mini",
"trinity-large", # prefix-only must not match
"claude-sonnet-4.6",
"gpt-5.4",
],
)
def test_is_arcee_trinity_thinking_rejects_non_matches(model) -> None:
assert _is_arcee_trinity_thinking(model) is False
def test_fixed_temperature_for_trinity_thinking() -> None:
assert _fixed_temperature_for_model("trinity-large-thinking") == 0.5
assert _fixed_temperature_for_model("arcee-ai/trinity-large-thinking") == 0.5
def test_fixed_temperature_sibling_arcee_models_unaffected() -> None:
# Preview and mini do not pin temperature — caller chooses its default.
assert _fixed_temperature_for_model("trinity-large-preview") is None
assert _fixed_temperature_for_model("trinity-mini") is None
def test_compression_threshold_for_trinity_thinking() -> None:
assert _compression_threshold_for_model("trinity-large-thinking") == 0.75
assert _compression_threshold_for_model("arcee-ai/trinity-large-thinking") == 0.75
def test_compression_threshold_default_none_for_other_models() -> None:
# None means "leave the user's config value unchanged".
assert _compression_threshold_for_model(None) is None
assert _compression_threshold_for_model("") is None
assert _compression_threshold_for_model("trinity-large-preview") is None
assert _compression_threshold_for_model("claude-sonnet-4.6") is None
assert _compression_threshold_for_model("kimi-k2") is None
+156
View File
@@ -0,0 +1,156 @@
"""Tests for agent.async_utils.safe_schedule_threadsafe."""
from __future__ import annotations
import asyncio
import gc
import warnings
from concurrent.futures import Future
from unittest.mock import patch
from agent.async_utils import safe_schedule_threadsafe
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _no_unawaited_warnings(caught, *, coro_name: str = "") -> bool:
"""Return True if no "X was never awaited" warning slipped through.
When *coro_name* is provided, only warnings naming that coroutine are
counted — xdist workers may emit unrelated unawaited-coroutine warnings
(e.g. ``AsyncMockMixin._execute_mock_call``) from concurrent tests.
"""
bad = [
w for w in caught
if issubclass(w.category, RuntimeWarning)
and "was never awaited" in str(w.message)
and (not coro_name or coro_name in str(w.message))
]
return not bad
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestSafeScheduleThreadsafe:
def test_returns_future_on_success(self):
loop = asyncio.new_event_loop()
try:
import threading
ready = threading.Event()
stop = threading.Event()
def _runner():
asyncio.set_event_loop(loop)
ready.set()
loop.run_until_complete(_wait_for_stop(stop))
async def _wait_for_stop(ev):
while not ev.is_set():
await asyncio.sleep(0.005)
t = threading.Thread(target=_runner, daemon=True)
t.start()
ready.wait(timeout=2)
async def _sample():
return 42
fut = safe_schedule_threadsafe(_sample(), loop)
assert isinstance(fut, Future)
assert fut.result(timeout=2) == 42
stop.set()
t.join(timeout=2)
finally:
if loop.is_running():
loop.call_soon_threadsafe(loop.stop)
loop.close()
def test_closed_loop_returns_none_and_closes_coroutine(self):
loop = asyncio.new_event_loop()
loop.close()
async def _sample():
return "ok"
coro = _sample()
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
result = safe_schedule_threadsafe(coro, loop)
del coro
gc.collect()
assert result is None
assert _no_unawaited_warnings(caught, coro_name='_sample')
def test_none_loop_returns_none_and_closes_coroutine(self):
async def _sample():
return "ok"
coro = _sample()
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
result = safe_schedule_threadsafe(coro, None)
del coro
gc.collect()
assert result is None
assert _no_unawaited_warnings(caught, coro_name='_sample')
def test_scheduling_exception_closes_coroutine(self):
"""If run_coroutine_threadsafe raises, close the coroutine and return None."""
# A loop that *looks* open but raises on submission
loop = asyncio.new_event_loop()
try:
async def _sample():
return "ok"
coro = _sample()
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with patch(
"agent.async_utils.asyncio.run_coroutine_threadsafe",
side_effect=RuntimeError("scheduler down"),
):
result = safe_schedule_threadsafe(coro, loop)
del coro
gc.collect()
assert result is None
assert _no_unawaited_warnings(caught, coro_name='_sample')
finally:
loop.close()
def test_logs_at_specified_level(self, caplog):
import logging
loop = asyncio.new_event_loop()
loop.close()
async def _sample():
return None
custom = logging.getLogger("test_async_utils")
with caplog.at_level(logging.WARNING, logger="test_async_utils"):
result = safe_schedule_threadsafe(
_sample(), loop,
logger=custom,
log_message="custom-msg",
log_level=logging.WARNING,
)
assert result is None
assert any("custom-msg" in rec.message for rec in caplog.records)
def test_non_coroutine_arg_does_not_crash(self):
"""Defensive: even if the caller hands us something weird, don't blow up."""
loop = asyncio.new_event_loop()
loop.close()
# Pass a non-coroutine sentinel
result = safe_schedule_threadsafe("not-a-coroutine", loop) # type: ignore[arg-type]
assert result is None
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,107 @@
"""Tests for agent.auxiliary_client._try_custom_endpoint's anthropic_messages branch.
When a user configures a custom endpoint with ``api_mode: anthropic_messages``
(e.g. MiniMax, Zhipu GLM, LiteLLM in Anthropic-proxy mode), auxiliary tasks
(compression, web_extract, session_search, title generation) must use the
native Anthropic transport rather than being silently downgraded to an
OpenAI-wire client that speaks the wrong protocol.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch):
for key in (
"OPENAI_API_KEY", "OPENAI_BASE_URL",
"ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN",
):
monkeypatch.delenv(key, raising=False)
def _install_anthropic_adapter_mocks():
"""Patch build_anthropic_client so the test doesn't need the SDK."""
fake_client = MagicMock(name="anthropic_client")
return patch(
"agent.anthropic_adapter.build_anthropic_client",
return_value=fake_client,
), fake_client
def test_custom_endpoint_anthropic_messages_builds_anthropic_wrapper():
"""api_mode=anthropic_messages → returns AnthropicAuxiliaryClient, not OpenAI."""
from agent.auxiliary_client import _try_custom_endpoint, AnthropicAuxiliaryClient
with patch(
"agent.auxiliary_client._resolve_custom_runtime",
return_value=(
"https://api.minimax.io/anthropic",
"minimax-key",
"anthropic_messages",
),
), patch(
"agent.auxiliary_client._read_main_model",
return_value="claude-sonnet-4-6",
):
adapter_patch, fake_client = _install_anthropic_adapter_mocks()
with adapter_patch:
client, model = _try_custom_endpoint()
assert isinstance(client, AnthropicAuxiliaryClient), (
"Custom endpoint with api_mode=anthropic_messages must return the "
f"native Anthropic wrapper, got {type(client).__name__}"
)
assert model == "claude-sonnet-4-6"
# Wrapper should NOT be marked as OAuth — third-party endpoints are
# always API-key authenticated.
assert client.api_key == "minimax-key"
assert client.base_url == "https://api.minimax.io/anthropic"
def test_custom_endpoint_anthropic_messages_falls_back_when_sdk_missing():
"""Graceful degradation when anthropic SDK is unavailable."""
from agent.auxiliary_client import _try_custom_endpoint
import_error = ImportError("anthropic package not installed")
with patch(
"agent.auxiliary_client._resolve_custom_runtime",
return_value=("https://api.minimax.io/anthropic", "k", "anthropic_messages"),
), patch(
"agent.auxiliary_client._read_main_model",
return_value="claude-sonnet-4-6",
), patch(
"agent.anthropic_adapter.build_anthropic_client",
side_effect=import_error,
):
client, model = _try_custom_endpoint()
# Should fall back to an OpenAI-wire client rather than returning
# (None, None) — the tool still needs to do *something*.
assert client is not None
assert model == "claude-sonnet-4-6"
# OpenAI client, not AnthropicAuxiliaryClient.
from agent.auxiliary_client import AnthropicAuxiliaryClient
assert not isinstance(client, AnthropicAuxiliaryClient)
def test_custom_endpoint_chat_completions_still_uses_openai_wire():
"""Regression: default path (no api_mode) must remain OpenAI client."""
from agent.auxiliary_client import _try_custom_endpoint, AnthropicAuxiliaryClient
with patch(
"agent.auxiliary_client._resolve_custom_runtime",
return_value=("https://api.example.com/v1", "key", None),
), patch(
"agent.auxiliary_client._read_main_model",
return_value="my-model",
):
client, model = _try_custom_endpoint()
assert client is not None
assert model == "my-model"
assert not isinstance(client, AnthropicAuxiliaryClient)
@@ -0,0 +1,349 @@
"""Tests for auxiliary client routing of the ``azure-foundry`` provider.
Covers the dedicated branch in ``agent.auxiliary_client.resolve_provider_client``
that delegates to :func:`hermes_cli.runtime_provider._resolve_azure_foundry_runtime`
instead of falling into the generic ``resolve_api_key_provider_credentials``
path (which only knows about ``AZURE_FOUNDRY_API_KEY`` and would 401 for
Entra ID users and miss ``model.base_url`` overrides for api-key users
with non-standard Foundry-projects endpoints).
Pinned scenarios:
* ``auth_mode: api_key`` → plain OpenAI client with the static string
key for ``chat_completions``.
* ``auth_mode: entra_id`` + ``chat_completions`` → plain OpenAI
client with a callable ``api_key`` (the bearer-token provider) —
confirms the callable survives the auxiliary path end-to-end.
* ``auth_mode: entra_id`` + GPT-5.x model → CodexAuxiliaryClient
wrapping the OpenAI client (api_mode auto-upgrades to
codex_responses).
* Anthropic-style + entra_id → rejected at the runtime resolver,
so the aux path returns ``(None, None)``.
* Failure path when no model is configured returns ``(None, None)``
cleanly so the auto chain falls through.
"""
from __future__ import annotations
import sys
from types import SimpleNamespace
import pytest
@pytest.fixture(autouse=True)
def _reset_credential_cache():
from agent.azure_identity_adapter import reset_credential_cache
reset_credential_cache()
yield
reset_credential_cache()
@pytest.fixture
def fake_azure_identity(monkeypatch):
"""Stand-in for azure.identity (keeps CI hermetic when the SDK is
not installed)."""
from agent import azure_identity_adapter as _adapter
last = {"scope": None}
def _provider(scope):
return lambda: f"jwt-for-{scope}"
fake_module = SimpleNamespace(
DefaultAzureCredential=lambda **kw: SimpleNamespace(
kwargs=kw,
get_token=lambda scope: SimpleNamespace(token="fake", expires_on=9999999999),
),
get_bearer_token_provider=lambda credential, scope: (
last.__setitem__("scope", scope),
_provider(scope),
)[-1],
)
monkeypatch.setattr(_adapter, "_require_azure_identity", lambda: fake_module)
monkeypatch.setitem(sys.modules, "azure.identity", fake_module)
return last
@pytest.fixture
def patch_load_config(monkeypatch):
"""Helper to set model_cfg seen by _try_azure_foundry."""
def _apply(model_cfg):
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"model": model_cfg},
)
return _apply
# ---------------------------------------------------------------------------
# auth_mode: api_key (default) — regression for the legacy path
# ---------------------------------------------------------------------------
class TestAuxAzureFoundryApiKey:
def test_chat_completions_returns_plain_openai_client(self, monkeypatch, patch_load_config):
from agent.auxiliary_client import _try_azure_foundry
from openai import OpenAI as _OpenAI
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "sk-azure-static-key")
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
"default": "gpt-4o",
})
client, resolved = _try_azure_foundry(model="gpt-4o")
assert client is not None
assert resolved == "gpt-4o"
assert isinstance(client, _OpenAI)
assert client.api_key == "sk-azure-static-key"
def test_codex_responses_wraps_in_codex_aux_client(self, monkeypatch, patch_load_config):
from agent.auxiliary_client import _try_azure_foundry, CodexAuxiliaryClient
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "sk-azure-static-key")
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
"default": "gpt-5.4-mini",
})
# GPT-5.x → runtime auto-upgrades to codex_responses
client, resolved = _try_azure_foundry(model="gpt-5.4-mini")
assert resolved == "gpt-5.4-mini"
assert isinstance(client, CodexAuxiliaryClient)
assert client.api_key == "sk-azure-static-key"
def test_no_key_returns_none(self, monkeypatch, patch_load_config):
from agent.auxiliary_client import _try_azure_foundry
monkeypatch.delenv("AZURE_FOUNDRY_API_KEY", raising=False)
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
"default": "gpt-4o",
})
client, resolved = _try_azure_foundry(model="gpt-4o")
assert client is None
assert resolved is None
def test_no_model_returns_none(self, monkeypatch, patch_load_config):
"""Azure has no fallback aux model — fail soft so the auto chain
can try other providers."""
from agent.auxiliary_client import _try_azure_foundry
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "sk-azure-static-key")
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
# No default model
})
client, resolved = _try_azure_foundry()
assert client is None
assert resolved is None
# ---------------------------------------------------------------------------
# auth_mode: entra_id — callable api_key survives end-to-end
# ---------------------------------------------------------------------------
class TestAuxAzureFoundryEntra:
def test_callable_api_key_reaches_openai_constructor(
self, monkeypatch, fake_azure_identity, patch_load_config,
):
"""The token provider callable must arrive at ``OpenAI(api_key=...)``
intact — never stringified to ``"no-key-required"`` or to the
SDK-internal empty-string representation BEFORE we hand it off.
We assert on the public SDK contract (constructor receives the
callable) rather than ``client.api_key``, because OpenAI 2.24.0
stores callable api_keys in a private attribute and exposes
``client.api_key`` as ``""``. The SDK still calls the callable
per request to mint ``Authorization: Bearer <token>``; that
behaviour is the documented Microsoft/OpenAI contract we rely on.
"""
from agent import auxiliary_client as _aux
received = {}
class _FakeOpenAI:
def __init__(self, **kwargs):
received.update(kwargs)
# Mirror the fields downstream callers read.
self.api_key = kwargs.get("api_key", "")
self.base_url = kwargs.get("base_url", "")
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
"auth_mode": "entra_id",
"default": "gpt-4o",
})
client, resolved = _aux._try_azure_foundry(model="gpt-4o")
assert client is not None
assert resolved == "gpt-4o"
# Public-contract assertion: the OpenAI SDK constructor saw the
# callable, exactly as Microsoft's Foundry sample requires.
assert callable(received["api_key"])
assert not isinstance(received["api_key"], str)
assert received["api_key"]().startswith("jwt-for-")
# Base URL forwarded verbatim (no /responses suffix stripping
# in this path — that's a separate concern handled by the
# runtime resolver only when the user re-saves config).
assert received["base_url"] == "https://r.openai.azure.com/openai/v1"
def test_codex_responses_with_entra_wraps_correctly(
self, monkeypatch, fake_azure_identity, patch_load_config,
):
"""GPT-5.x deployment on Entra ID — auto-upgraded to
codex_responses, wrapped in CodexAuxiliaryClient, callable
api_key handed to the underlying OpenAI SDK."""
from agent import auxiliary_client as _aux
received = {}
class _FakeOpenAI:
def __init__(self, **kwargs):
received.update(kwargs)
self.api_key = kwargs.get("api_key", "")
self.base_url = kwargs.get("base_url", "")
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
"auth_mode": "entra_id",
"default": "gpt-5.4-mini",
})
client, resolved = _aux._try_azure_foundry(model="gpt-5.4-mini")
assert resolved == "gpt-5.4-mini"
assert isinstance(client, _aux.CodexAuxiliaryClient)
# The Codex wrapper received an OpenAI client built with the
# callable api_key — verify against the SDK constructor record,
# not the wrapper attribute (which mirrors the SDK's empty-
# string representation).
assert callable(received["api_key"])
assert received["api_key"]().startswith("jwt-for-")
def test_entra_anthropic_messages_uses_bearer_hook(
self, monkeypatch, fake_azure_identity, patch_load_config,
):
"""Entra ID + anthropic_messages: runtime returns a callable
api_key; ``_maybe_wrap_anthropic`` → ``build_anthropic_client``
detects the callable and installs the bearer-injecting httpx
event hook on a custom ``httpx.Client`` passed to the
Anthropic SDK via ``http_client=``."""
from agent import auxiliary_client as _aux
from agent import anthropic_adapter as _anthropic
received = {}
class _FakeOpenAI:
def __init__(self, **kwargs):
received["openai"] = kwargs
self.api_key = kwargs.get("api_key", "")
self.base_url = kwargs.get("base_url", "")
class _FakeAnthropicSDK:
class Anthropic:
def __init__(self, **kwargs):
received["anthropic"] = kwargs
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
monkeypatch.setattr(_anthropic, "_get_anthropic_sdk", lambda: _FakeAnthropicSDK)
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.services.ai.azure.com/anthropic",
"api_mode": "anthropic_messages",
"auth_mode": "entra_id",
"default": "claude-sonnet-4-5",
})
client, resolved = _aux._try_azure_foundry(model="claude-sonnet-4-5")
assert client is not None
assert resolved == "claude-sonnet-4-5"
# The Anthropic SDK constructor received a custom http_client
# (the bearer-injecting hook) and a placeholder auth_token.
anthropic_kwargs = received.get("anthropic") or {}
assert "http_client" in anthropic_kwargs, (
"build_anthropic_client must pass a custom http_client when "
"given a callable api_key, otherwise the SDK cannot mint "
"fresh tokens per request"
)
assert anthropic_kwargs.get("auth_token") == "entra-id-bearer-via-http-hook"
# Verify the http_client actually has our event hook installed.
http_client = anthropic_kwargs["http_client"]
hooks = getattr(http_client, "event_hooks", {})
assert "request" in hooks and len(hooks["request"]) >= 1
# ---------------------------------------------------------------------------
# resolve_provider_client → azure-foundry dispatch
# ---------------------------------------------------------------------------
class TestResolveProviderClientAzureFoundry:
def test_dispatches_to_azure_branch_not_generic_api_key_path(
self, monkeypatch, fake_azure_identity, patch_load_config,
):
"""End-to-end: the public ``resolve_provider_client`` entry
point must take the dedicated azure-foundry branch, NOT the
generic api-key registry path that would call
``resolve_api_key_provider_credentials`` and return None for
Entra users."""
from agent import auxiliary_client as _aux
received = {}
class _FakeOpenAI:
def __init__(self, **kwargs):
received.update(kwargs)
self.api_key = kwargs.get("api_key", "")
self.base_url = kwargs.get("base_url", "")
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
"auth_mode": "entra_id",
"default": "gpt-4o",
})
client, resolved = _aux.resolve_provider_client("azure-foundry", "gpt-4o")
assert client is not None
assert resolved == "gpt-4o"
# The callable made it through resolve_provider_client → _try_azure_foundry
# → OpenAI(api_key=...).
assert callable(received["api_key"])
def test_warns_and_returns_none_on_failure(
self, monkeypatch, patch_load_config, caplog,
):
"""When azure-foundry is requested but cannot be resolved
(e.g. no model + no key), we return (None, None) and log a
clear warning pointing at ``hermes doctor``."""
import logging
from agent.auxiliary_client import resolve_provider_client
monkeypatch.delenv("AZURE_FOUNDRY_API_KEY", raising=False)
patch_load_config({
"provider": "azure-foundry",
"base_url": "https://r.openai.azure.com/openai/v1",
"api_mode": "chat_completions",
# No default → resolver yields no model → bail
})
with caplog.at_level(logging.WARNING, logger="agent.auxiliary_client"):
client, resolved = resolve_provider_client("azure-foundry")
assert client is None
assert resolved is None
assert any(
"azure-foundry" in rec.message and "hermes doctor" in rec.message
for rec in caplog.records
)
@@ -0,0 +1,153 @@
"""Tests for xAI OAuth 403 error recovery in auxiliary_client.
xAI returns HTTP 403 (not 401) with "unauthenticated:bad-credentials" when
an OAuth2 access token has expired. These tests verify the three fixes:
1. _is_auth_error detects xAI 403 as an auth failure
2. _recoverable_pool_provider maps api.x.ai to xai-oauth
3. _refresh_provider_credentials includes xai-oauth refresh logic
"""
import pytest
# ── _is_auth_error ──────────────────────────────────────────────────────────
def _import_is_auth_error():
from agent.auxiliary_client import _is_auth_error
return _is_auth_error
class TestIsAuthErrorXaiOauth403:
"""Verify _is_auth_error correctly identifies xAI's 403 bad-credentials."""
@pytest.fixture(autouse=True)
def _import(self):
self.is_auth_error = _import_is_auth_error()
def test_xai_403_bad_credentials_is_auth_error(self):
"""The exact error xAI returns for expired OAuth tokens."""
exc = Exception(
"Error code: 403 - {'code': 'The caller does not have permission "
"to execute the specified operation', 'error': 'The OAuth2 access "
"token could not be validated. [WKE=unauthenticated:bad-credentials]'}"
)
exc.status_code = 403 # openai.PermissionDenied sets this
assert self.is_auth_error(exc) is True
def test_xai_403_bad_credentials_without_status_code(self):
"""Fallback match when status_code attribute is missing."""
exc = Exception(
"Error code: 403 - unauthenticated:bad-credentials"
)
# No status_code attribute — should still match via string pattern
assert self.is_auth_error(exc) is True
def test_generic_403_is_not_auth_error(self):
"""A generic 403 (e.g. rate limit, forbidden) should NOT be treated as auth."""
exc = Exception("Error code: 403 - rate limit exceeded")
exc.status_code = 403
assert self.is_auth_error(exc) is False
def test_401_status_code_is_auth_error(self):
"""Existing 401 detection still works."""
exc = Exception("Unauthorized")
exc.status_code = 401
assert self.is_auth_error(exc) is True
def test_401_string_is_auth_error(self):
"""Existing string-based 401 detection still works."""
exc = Exception("Error code: 401 - Unauthorized")
assert self.is_auth_error(exc) is True
def test_authentication_error_class_is_auth_error(self):
"""Existing AuthenticationError class detection still works."""
exc_type = type("AuthenticationError", (Exception,), {})
exc = exc_type("auth failure")
assert self.is_auth_error(exc) is True
def test_permission_denied_without_bad_credentials_is_not_auth_error(self):
"""403 PermissionDenied without bad-credentials should not be auth."""
exc = Exception("Error code: 403 - Permission denied")
exc.status_code = 403
assert self.is_auth_error(exc) is False
def test_500_is_not_auth_error(self):
"""Server errors are not auth errors."""
exc = Exception("Error code: 500 - Internal server error")
exc.status_code = 500
assert self.is_auth_error(exc) is False
def test_unauthenticated_without_bad_credentials_is_not_auth_error(self):
"""'unauthenticated' alone (without 'bad-credentials') should not match."""
exc = Exception("unauthenticated request")
assert self.is_auth_error(exc) is False
# ── _recoverable_pool_provider ──────────────────────────────────────────────
def _import_recoverable_pool_provider():
from agent.auxiliary_client import _recoverable_pool_provider
return _recoverable_pool_provider
class TestRecoverablePoolProviderXaiOAuth:
"""Verify _recoverable_pool_provider maps api.x.ai to xai-oauth."""
@pytest.fixture(autouse=True)
def _import(self):
self.recover = _import_recoverable_pool_provider()
def test_explicit_xai_oauth_provider(self):
"""Explicit provider name passes through."""
result = self.recover("xai-oauth", None)
assert result == "xai-oauth"
def test_api_x_ai_host_match(self):
"""api.x.ai base URL maps to xai-oauth pool."""
class MockClient:
base_url = "https://api.x.ai/v1/"
result = self.recover("auto", MockClient())
assert result == "xai-oauth"
def test_auto_with_unknown_host_returns_none(self):
"""auto provider with unknown host returns None."""
class MockClient:
base_url = "https://unknown.example.com/v1/"
result = self.recover("auto", MockClient())
assert result is None
# ── _refresh_provider_credentials (structure check) ─────────────────────────
def _import_refresh_provider_credentials():
from agent.auxiliary_client import _refresh_provider_credentials
return _refresh_provider_credentials
class TestRefreshProviderCredentialsXaiOAuth:
"""Verify _refresh_provider_credentials has xai-oauth branch.
Full integration testing requires live OAuth tokens, so we verify
the branch exists and handles the no-credential case gracefully.
"""
@pytest.fixture(autouse=True)
def _import(self):
self.refresh = _import_refresh_provider_credentials()
def test_xai_oauth_no_pool_returns_false(self):
"""When no xai-oauth pool exists, refresh returns False gracefully."""
# This tests that the branch exists and doesn't crash.
# It may return True if the singleton resolver finds tokens,
# or False if neither pool nor singleton has credentials.
# Either way, it should not raise an exception.
result = self.refresh("xai-oauth")
assert isinstance(result, bool)
def test_unknown_provider_returns_false(self):
"""Unknown providers fall through to return False."""
result = self.refresh("unknown-provider-xyz")
assert result is False
+310
View File
@@ -0,0 +1,310 @@
"""Tests for auxiliary model config bridging — verifies that config.yaml values
are properly mapped to environment variables by both CLI and gateway loaders.
Also tests the vision_tools and browser_tool model override env vars.
"""
import os
import sys
from pathlib import Path
from unittest.mock import patch, MagicMock
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
def _run_auxiliary_bridge(config_dict, monkeypatch):
"""Simulate the auxiliary config → env var bridging logic shared by CLI and gateway.
This mirrors the code in cli.py load_cli_config() and gateway/run.py.
Both use the same pattern; we test it once here.
"""
# Clear env vars
for key in (
"AUXILIARY_VISION_PROVIDER", "AUXILIARY_VISION_MODEL",
"AUXILIARY_VISION_BASE_URL", "AUXILIARY_VISION_API_KEY",
"AUXILIARY_WEB_EXTRACT_PROVIDER", "AUXILIARY_WEB_EXTRACT_MODEL",
"AUXILIARY_WEB_EXTRACT_BASE_URL", "AUXILIARY_WEB_EXTRACT_API_KEY",
):
monkeypatch.delenv(key, raising=False)
# Compression config is read directly from config.yaml — no env var bridging.
# Auxiliary bridge
auxiliary_cfg = config_dict.get("auxiliary", {})
if auxiliary_cfg and isinstance(auxiliary_cfg, dict):
aux_task_env = {
"vision": {
"provider": "AUXILIARY_VISION_PROVIDER",
"model": "AUXILIARY_VISION_MODEL",
"base_url": "AUXILIARY_VISION_BASE_URL",
"api_key": "AUXILIARY_VISION_API_KEY",
},
"web_extract": {
"provider": "AUXILIARY_WEB_EXTRACT_PROVIDER",
"model": "AUXILIARY_WEB_EXTRACT_MODEL",
"base_url": "AUXILIARY_WEB_EXTRACT_BASE_URL",
"api_key": "AUXILIARY_WEB_EXTRACT_API_KEY",
},
}
for task_key, env_map in aux_task_env.items():
task_cfg = auxiliary_cfg.get(task_key, {})
if not isinstance(task_cfg, dict):
continue
prov = str(task_cfg.get("provider", "")).strip()
model = str(task_cfg.get("model", "")).strip()
base_url = str(task_cfg.get("base_url", "")).strip()
api_key = str(task_cfg.get("api_key", "")).strip()
if prov and prov != "auto":
os.environ[env_map["provider"]] = prov
if model:
os.environ[env_map["model"]] = model
if base_url:
os.environ[env_map["base_url"]] = base_url
if api_key:
os.environ[env_map["api_key"]] = api_key
# ── Config bridging tests ────────────────────────────────────────────────────
class TestAuxiliaryConfigBridge:
"""Verify the config.yaml → env var bridging logic used by CLI and gateway."""
def test_vision_provider_bridged(self, monkeypatch):
config = {
"auxiliary": {
"vision": {"provider": "openrouter", "model": ""},
"web_extract": {"provider": "auto", "model": ""},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
# auto should not be set
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None
def test_vision_model_bridged(self, monkeypatch):
config = {
"auxiliary": {
"vision": {"provider": "auto", "model": "openai/gpt-4o"},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_MODEL") == "openai/gpt-4o"
# auto provider should not be set
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
def test_web_extract_bridged(self, monkeypatch):
config = {
"auxiliary": {
"web_extract": {"provider": "nous", "model": "gemini-2.5-flash"},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") == "nous"
assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "gemini-2.5-flash"
def test_direct_endpoint_bridged(self, monkeypatch):
config = {
"auxiliary": {
"vision": {
"base_url": "http://localhost:1234/v1",
"api_key": "local-key",
"model": "qwen2.5-vl",
}
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_BASE_URL") == "http://localhost:1234/v1"
assert os.environ.get("AUXILIARY_VISION_API_KEY") == "local-key"
assert os.environ.get("AUXILIARY_VISION_MODEL") == "qwen2.5-vl"
def test_empty_values_not_bridged(self, monkeypatch):
config = {
"auxiliary": {
"vision": {"provider": "auto", "model": ""},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
assert os.environ.get("AUXILIARY_VISION_MODEL") is None
def test_missing_auxiliary_section_safe(self, monkeypatch):
"""Config without auxiliary section should not crash."""
config = {"model": {"default": "test-model"}}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
def test_non_dict_task_config_ignored(self, monkeypatch):
"""Malformed task config (e.g. string instead of dict) is safely ignored."""
config = {
"auxiliary": {
"vision": "openrouter", # should be a dict
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
def test_mixed_tasks(self, monkeypatch):
config = {
"auxiliary": {
"vision": {"provider": "openrouter", "model": ""},
"web_extract": {"provider": "auto", "model": "custom-llm"},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
assert os.environ.get("AUXILIARY_VISION_MODEL") is None
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None
assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "custom-llm"
def test_all_tasks_with_overrides(self, monkeypatch):
config = {
"auxiliary": {
"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"},
"web_extract": {"provider": "nous", "model": "gemini-3-flash"},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
assert os.environ.get("AUXILIARY_VISION_MODEL") == "google/gemini-2.5-flash"
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") == "nous"
assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "gemini-3-flash"
def test_whitespace_in_values_stripped(self, monkeypatch):
config = {
"auxiliary": {
"vision": {"provider": " openrouter ", "model": " my-model "},
}
}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
assert os.environ.get("AUXILIARY_VISION_MODEL") == "my-model"
def test_empty_auxiliary_dict_safe(self, monkeypatch):
config = {"auxiliary": {}}
_run_auxiliary_bridge(config, monkeypatch)
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None
# ── Gateway bridge parity test ───────────────────────────────────────────────
class TestGatewayBridgeCodeParity:
"""Verify the gateway/run.py config bridge contains the auxiliary section."""
def test_gateway_has_auxiliary_bridge(self):
"""The gateway config bridge must include auxiliary.* bridging.
After the plugin-aux-task API refactor (2026-05), gateway env-var
names are derived dynamically (``AUXILIARY_<KEY_UPPER>_*``) so the
literal strings ``AUXILIARY_VISION_PROVIDER`` etc. no longer appear
in source. Assert the dynamic shape and the canonical built-in keys
bridged set instead.
"""
gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py"
# Pin encoding to UTF-8: source files in this repo are UTF-8, but
# Path.read_text() defaults to the system locale — which is cp1252
# on most Western Windows installs and crashes as soon as the file
# contains any non-ASCII byte (e.g. an em-dash in a comment).
content = gateway_path.read_text(encoding="utf-8")
# Dynamic env-var derivation present
assert 'f"AUXILIARY_{_upper}_PROVIDER"' in content
assert 'f"AUXILIARY_{_upper}_MODEL"' in content
assert 'f"AUXILIARY_{_upper}_BASE_URL"' in content
assert 'f"AUXILIARY_{_upper}_API_KEY"' in content
# Built-in bridged keys present
assert "_aux_bridged_keys" in content
assert '"vision"' in content
assert '"web_extract"' in content
assert '"approval"' in content
# Plugin-aux-task discovery hooked into bridging
assert "get_plugin_auxiliary_tasks" in content
def test_gateway_no_compression_env_bridge(self):
"""Gateway should NOT bridge compression config to env vars (config-only)."""
gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py"
# See note in test_gateway_has_auxiliary_bridge — pin UTF-8 so the
# test runs on Windows where the default locale is cp1252.
content = gateway_path.read_text(encoding="utf-8")
assert "CONTEXT_COMPRESSION_PROVIDER" not in content
assert "CONTEXT_COMPRESSION_MODEL" not in content
# ── Vision model override tests ──────────────────────────────────────────────
class TestVisionModelOverride:
"""Test that AUXILIARY_VISION_MODEL env var overrides the default model in the handler."""
def test_env_var_overrides_default(self, monkeypatch):
monkeypatch.setenv("AUXILIARY_VISION_MODEL", "openai/gpt-4o")
from tools.vision_tools import _handle_vision_analyze
with patch("tools.vision_tools.vision_analyze_tool", new_callable=MagicMock) as mock_tool:
mock_tool.return_value = '{"success": true}'
_handle_vision_analyze({"image_url": "http://test.jpg", "question": "test"})
call_args = mock_tool.call_args
# 3rd positional arg = model
assert call_args[0][2] == "openai/gpt-4o"
def test_default_model_when_no_override(self, monkeypatch):
monkeypatch.delenv("AUXILIARY_VISION_MODEL", raising=False)
from tools.vision_tools import _handle_vision_analyze
with patch("tools.vision_tools.vision_analyze_tool", new_callable=MagicMock) as mock_tool:
mock_tool.return_value = '{"success": true}'
_handle_vision_analyze({"image_url": "http://test.jpg", "question": "test"})
call_args = mock_tool.call_args
# With no AUXILIARY_VISION_MODEL env var, model should be None
# (the centralized call_llm router picks the provider default)
assert call_args[0][2] is None
# ── DEFAULT_CONFIG shape tests ───────────────────────────────────────────────
class TestDefaultConfigShape:
"""Verify the DEFAULT_CONFIG in hermes_cli/config.py has correct auxiliary structure."""
def test_auxiliary_section_exists(self):
from hermes_cli.config import DEFAULT_CONFIG
assert "auxiliary" in DEFAULT_CONFIG
def test_vision_task_structure(self):
from hermes_cli.config import DEFAULT_CONFIG
vision = DEFAULT_CONFIG["auxiliary"]["vision"]
assert "provider" in vision
assert "model" in vision
assert vision["provider"] == "auto"
assert vision["model"] == ""
def test_web_extract_task_structure(self):
from hermes_cli.config import DEFAULT_CONFIG
web = DEFAULT_CONFIG["auxiliary"]["web_extract"]
assert "provider" in web
assert "model" in web
assert web["provider"] == "auto"
assert web["model"] == ""
# ── CLI defaults parity ─────────────────────────────────────────────────────
class TestCLIDefaultsHaveAuxiliaryKeys:
"""Verify cli.py load_cli_config() defaults dict does NOT include auxiliary
(it comes from config.yaml deep merge, not hardcoded defaults)."""
def test_cli_defaults_can_merge_auxiliary(self):
"""The load_cli_config deep merge logic handles keys not in defaults.
Verify auxiliary would be picked up from config.yaml."""
# This is a structural assertion: cli.py's second-pass loop
# carries over keys from file_config that aren't in defaults.
# So auxiliary config from config.yaml gets merged even though
# cli.py's defaults dict doesn't define it.
import cli as _cli_mod
# See note in test_gateway_has_auxiliary_bridge — pin UTF-8 so the
# test runs on Windows where the default locale is cp1252.
source = Path(_cli_mod.__file__).read_text(encoding="utf-8")
assert "auxiliary_config = defaults.get(\"auxiliary\"" in source
assert "AUXILIARY_VISION_PROVIDER" in source
assert "AUXILIARY_VISION_MODEL" in source
+413
View File
@@ -0,0 +1,413 @@
"""Regression tests for the ``auto`` → main-model-first policy.
Prior to this change, aggregator users (OpenRouter / Nous Portal) had aux
tasks routed through a cheap provider-side default (Gemini Flash) while
non-aggregator users got their main model. This made behavior inconsistent
and surprising — users picked Claude but got Gemini Flash summaries.
The current policy: ``auto`` means "use my main chat model" for every user,
regardless of provider type. Explicit per-task overrides in ``config.yaml``
(``auxiliary.<task>.provider``) still win. The cheap fallback chain only
runs when the main provider has no working client.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
# ── Text aux tasks — _resolve_auto ──────────────────────────────────────────
class TestResolveAutoMainFirst:
"""_resolve_auto() must prefer main provider + main model for every user."""
def test_openrouter_main_uses_main_model_for_aux(self, monkeypatch):
"""OpenRouter main user → aux uses their picked OR model, not Gemini Flash."""
monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key")
with patch(
"agent.auxiliary_client._read_main_provider",
return_value="openrouter",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="anthropic/claude-sonnet-4.6",
), patch(
"agent.auxiliary_client.resolve_provider_client"
) as mock_resolve:
mock_client = MagicMock()
mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6")
from agent.auxiliary_client import _resolve_auto
client, model = _resolve_auto()
assert client is mock_client
assert model == "anthropic/claude-sonnet-4.6"
# Verify it asked resolve_provider_client for the MAIN provider+model,
# not a fallback-chain provider
mock_resolve.assert_called_once()
assert mock_resolve.call_args.args[0] == "openrouter"
assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6"
def test_nous_main_uses_main_model_for_aux(self, monkeypatch):
"""Nous Portal main user → aux uses their picked Nous model, not free-tier MiMo."""
# No OPENROUTER_API_KEY → ensures if main failed we'd fall to chain
with patch(
"agent.auxiliary_client._read_main_provider", return_value="nous",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="anthropic/claude-opus-4.6",
), patch(
"agent.auxiliary_client.resolve_provider_client"
) as mock_resolve:
mock_client = MagicMock()
mock_resolve.return_value = (mock_client, "anthropic/claude-opus-4.6")
from agent.auxiliary_client import _resolve_auto
client, model = _resolve_auto()
assert client is mock_client
assert model == "anthropic/claude-opus-4.6"
assert mock_resolve.call_args.args[0] == "nous"
def test_non_aggregator_main_still_uses_main(self, monkeypatch):
"""Non-aggregator main (DeepSeek) → unchanged behavior, main model used."""
monkeypatch.setenv("DEEPSEEK_API_KEY", "ds-test")
with patch(
"agent.auxiliary_client._read_main_provider", return_value="deepseek",
), patch(
"agent.auxiliary_client._read_main_model", return_value="deepseek-chat",
), patch(
"agent.auxiliary_client.resolve_provider_client"
) as mock_resolve:
mock_client = MagicMock()
mock_resolve.return_value = (mock_client, "deepseek-chat")
from agent.auxiliary_client import _resolve_auto
client, model = _resolve_auto()
assert client is mock_client
assert model == "deepseek-chat"
assert mock_resolve.call_args.args[0] == "deepseek"
def test_main_unavailable_falls_through_to_chain(self, monkeypatch):
"""Main provider with no working client → fall back to aux chain."""
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
chain_client = MagicMock()
with patch(
"agent.auxiliary_client._read_main_provider", return_value="anthropic",
), patch(
"agent.auxiliary_client._read_main_model", return_value="claude-opus",
), patch(
"agent.auxiliary_client.resolve_provider_client",
return_value=(None, None), # main provider has no client
), patch(
"agent.auxiliary_client._try_openrouter",
return_value=(chain_client, "google/gemini-3-flash-preview"),
):
from agent.auxiliary_client import _resolve_auto
client, model = _resolve_auto()
assert client is chain_client
assert model == "google/gemini-3-flash-preview"
def test_no_main_config_uses_chain_directly(self):
"""No main provider configured → skip step 1, use chain (no regression)."""
chain_client = MagicMock()
with patch(
"agent.auxiliary_client._read_main_provider", return_value="",
), patch(
"agent.auxiliary_client._read_main_model", return_value="",
), patch(
"agent.auxiliary_client._try_openrouter",
return_value=(chain_client, "google/gemini-3-flash-preview"),
):
from agent.auxiliary_client import _resolve_auto
client, model = _resolve_auto()
assert client is chain_client
def test_runtime_override_wins_over_config(self, monkeypatch):
"""main_runtime kwarg overrides config-read main provider/model."""
with patch(
"agent.auxiliary_client._read_main_provider",
return_value="openrouter",
), patch(
"agent.auxiliary_client._read_main_model", return_value="config-model",
), patch(
"agent.auxiliary_client.resolve_provider_client"
) as mock_resolve:
mock_resolve.return_value = (MagicMock(), "runtime-model")
from agent.auxiliary_client import _resolve_auto
_resolve_auto(main_runtime={
"provider": "anthropic",
"model": "runtime-model",
"base_url": "",
"api_key": "",
"api_mode": "",
})
# Runtime override wins
assert mock_resolve.call_args.args[0] == "anthropic"
assert mock_resolve.call_args.args[1] == "runtime-model"
# ── Vision — resolve_vision_provider_client ─────────────────────────────────
class TestResolveVisionMainFirst:
"""Vision auto-detection prefers the main provider first."""
def test_openrouter_main_vision_uses_main_model(self, monkeypatch):
"""OpenRouter main with vision-capable model → aux vision uses main model."""
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
with patch(
"agent.auxiliary_client._read_main_provider", return_value="openrouter",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="anthropic/claude-sonnet-4.6",
), patch(
"agent.auxiliary_client.resolve_provider_client"
) as mock_resolve, patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
):
mock_client = MagicMock()
mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6")
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert provider == "openrouter"
assert client is mock_client
assert model == "anthropic/claude-sonnet-4.6"
# Verify it did NOT call the strict vision backend for OpenRouter
# (which would have used a cheap gemini-flash-preview default)
mock_resolve.assert_called_once()
assert mock_resolve.call_args.args[0] == "openrouter"
assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6"
assert mock_resolve.call_args.kwargs.get("is_vision") is True
def test_nous_main_vision_uses_paid_nous_vision_backend(self):
"""Paid Nous main → aux vision uses the dedicated Nous vision backend."""
with patch(
"agent.auxiliary_client._read_main_provider", return_value="nous",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="openai/gpt-5",
), patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
), patch(
"agent.auxiliary_client._resolve_strict_vision_backend",
return_value=(MagicMock(), "google/gemini-3-flash-preview"),
):
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert provider == "nous"
assert client is not None
assert model == "google/gemini-3-flash-preview"
def test_nous_main_vision_uses_free_tier_nous_vision_backend(self):
"""Free-tier Nous main → aux vision uses MiMo omni, not the text main model."""
with patch(
"agent.auxiliary_client._read_main_provider", return_value="nous",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="xiaomi/mimo-v2-pro",
), patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
), patch(
"agent.auxiliary_client._resolve_strict_vision_backend",
return_value=(MagicMock(), "xiaomi/mimo-v2-omni"),
):
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert provider == "nous"
assert client is not None
assert model == "xiaomi/mimo-v2-omni"
def test_exotic_provider_with_vision_override_preserved(self):
"""xiaomi → mimo-v2.5 override still wins over main_model."""
with patch(
"agent.auxiliary_client._read_main_provider", return_value="xiaomi",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="mimo-v2-pro", # text model
), patch(
"agent.auxiliary_client.resolve_provider_client"
) as mock_resolve, patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
):
mock_resolve.return_value = (MagicMock(), "mimo-v2.5")
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert provider == "xiaomi"
# Should use mimo-v2.5 (vision override), not mimo-v2-pro (text main)
assert mock_resolve.call_args.args[1] == "mimo-v2.5"
assert mock_resolve.call_args.kwargs.get("is_vision") is True
def test_copilot_vision_sets_vision_header(self, monkeypatch):
"""Copilot vision requests include the header required for vision routing."""
monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "ghu_test-token")
captured = {}
def fake_headers(*, is_agent_turn=False, is_vision=False):
captured["is_agent_turn"] = is_agent_turn
captured["is_vision"] = is_vision
return {"Copilot-Vision-Request": "true"} if is_vision else {}
with patch(
"agent.auxiliary_client._read_main_provider", return_value="copilot",
), patch(
"agent.auxiliary_client._read_main_model", return_value="configured-copilot-model",
), patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
), patch(
"agent.auxiliary_client.OpenAI",
) as mock_openai, patch(
"hermes_cli.auth.resolve_api_key_provider_credentials",
return_value={
"provider": "copilot",
"api_key": "copilot-api-token",
"base_url": "https://api.githubcopilot.com",
},
), patch(
"hermes_cli.copilot_auth.copilot_request_headers",
side_effect=fake_headers,
):
mock_client = MagicMock()
mock_openai.return_value = mock_client
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert provider == "copilot"
assert client is mock_client
assert model == "configured-copilot-model"
assert captured == {"is_agent_turn": True, "is_vision": True}
assert mock_openai.call_args.kwargs["default_headers"]["Copilot-Vision-Request"] == "true"
def test_text_copilot_does_not_set_vision_header(self, monkeypatch):
"""Text Copilot requests keep the vision-only header off."""
monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "ghu_test-token")
captured = {}
def fake_headers(*, is_agent_turn=False, is_vision=False):
captured["is_agent_turn"] = is_agent_turn
captured["is_vision"] = is_vision
return {"Copilot-Vision-Request": "true"} if is_vision else {}
with patch(
"agent.auxiliary_client.OpenAI",
) as mock_openai, patch(
"hermes_cli.auth.resolve_api_key_provider_credentials",
return_value={
"provider": "copilot",
"api_key": "copilot-api-token",
"base_url": "https://api.githubcopilot.com",
},
), patch(
"hermes_cli.copilot_auth.copilot_request_headers",
side_effect=fake_headers,
):
mock_client = MagicMock()
mock_openai.return_value = mock_client
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("copilot", "gpt-5-mini")
assert client is mock_client
assert model == "gpt-5-mini"
assert captured == {"is_agent_turn": True, "is_vision": False}
assert "default_headers" not in mock_openai.call_args.kwargs
def test_main_unavailable_vision_falls_through_to_aggregators(self):
"""Main provider fails → fall back to OpenRouter/Nous strict backends."""
fallback_client = MagicMock()
with patch(
"agent.auxiliary_client._read_main_provider", return_value="deepseek",
), patch(
"agent.auxiliary_client._read_main_model", return_value="deepseek-chat",
), patch(
"agent.auxiliary_client.resolve_provider_client",
return_value=(None, None),
), patch(
"agent.auxiliary_client._resolve_strict_vision_backend",
return_value=(fallback_client, "google/gemini-3-flash-preview"),
), patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
):
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert client is fallback_client
assert provider in {"openrouter", "nous"}
def test_explicit_provider_override_still_wins(self):
"""Explicit config override bypasses main-first policy."""
with patch(
"agent.auxiliary_client._read_main_provider", return_value="openrouter",
), patch(
"agent.auxiliary_client._read_main_model",
return_value="anthropic/claude-opus-4.6",
), patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("nous", None, None, None, None), # explicit override
), patch(
"agent.auxiliary_client._resolve_strict_vision_backend"
) as mock_strict:
mock_strict.return_value = (MagicMock(), "nous-default-model")
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
# Explicit "nous" override → uses strict backend, NOT main model path
assert provider == "nous"
mock_strict.assert_called_once_with("nous", None)
# ── Constant cleanup ────────────────────────────────────────────────────────
def test_aggregator_providers_constant_removed():
"""The dead _AGGREGATOR_PROVIDERS constant should no longer live in the module.
Removed when the main-first policy made the aggregator-skip guard obsolete.
"""
import agent.auxiliary_client as aux_mod
assert not hasattr(aux_mod, "_AGGREGATOR_PROVIDERS"), (
"_AGGREGATOR_PROVIDERS was removed when _resolve_auto stopped "
"treating aggregators specially. If you re-added it, the main-first "
"policy may have regressed."
)
@@ -0,0 +1,493 @@
"""Tests for named custom provider and 'main' alias resolution in auxiliary_client."""
from unittest.mock import patch, MagicMock
import pytest
@pytest.fixture(autouse=True)
def _isolate(tmp_path, monkeypatch):
"""Redirect HERMES_HOME and clear module caches."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
# Write a minimal config so load_config doesn't fail
(hermes_home / "config.yaml").write_text("model:\n default: test-model\n")
def _write_config(tmp_path, config_dict):
"""Write a config.yaml to the test HERMES_HOME."""
import yaml
config_path = tmp_path / ".hermes" / "config.yaml"
config_path.write_text(yaml.dump(config_dict))
class TestNormalizeVisionProvider:
"""_normalize_vision_provider should resolve 'main' to actual main provider."""
def test_main_resolves_to_named_custom(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "my-model", "provider": "custom:beans"},
"custom_providers": [{"name": "beans", "base_url": "http://localhost/v1"}],
})
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("main") == "custom:beans"
def test_main_resolves_to_openrouter(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "anthropic/claude-sonnet-4", "provider": "openrouter"},
})
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("main") == "openrouter"
def test_main_resolves_to_deepseek(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "deepseek-chat", "provider": "deepseek"},
})
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("main") == "deepseek"
def test_main_falls_back_to_custom_when_no_provider(self, tmp_path):
_write_config(tmp_path, {"model": {"default": "gpt-4o"}})
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("main") == "custom"
def test_bare_provider_name_unchanged(self):
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("beans") == "beans"
assert _normalize_vision_provider("deepseek") == "deepseek"
def test_custom_colon_named_provider_preserved(self):
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("custom:beans") == "beans"
def test_codex_alias_still_works(self):
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("codex") == "openai-codex"
def test_auto_unchanged(self):
from agent.auxiliary_client import _normalize_vision_provider
assert _normalize_vision_provider("auto") == "auto"
assert _normalize_vision_provider(None) == "auto"
class TestResolveProviderClientMainAlias:
"""resolve_provider_client('main', ...) should resolve to actual main provider."""
def test_main_resolves_to_named_custom_provider(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "my-model", "provider": "beans"},
"custom_providers": [
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
],
})
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("main", "override-model")
assert client is not None
assert model == "override-model"
assert "beans.local" in str(client.base_url)
def test_main_with_custom_colon_prefix(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "my-model", "provider": "custom:beans"},
"custom_providers": [
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
],
})
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("main", "test")
assert client is not None
assert "beans.local" in str(client.base_url)
def test_main_resolves_github_copilot_alias(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "gpt-5.4", "provider": "github-copilot"},
})
with (
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
"api_key": "ghu_test_token",
"base_url": "https://api.githubcopilot.com",
}),
patch("agent.auxiliary_client.OpenAI") as mock_openai,
):
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("main", "gpt-5.4")
assert client is not None
assert model == "gpt-5.4"
assert mock_openai.called
class TestResolveProviderClientNamedCustom:
"""resolve_provider_client should resolve named custom providers directly."""
def test_named_custom_provider(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "test-model"},
"custom_providers": [
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
],
})
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("beans", "my-model")
assert client is not None
assert model == "my-model"
assert "beans.local" in str(client.base_url)
def test_named_custom_provider_default_model(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "main-model"},
"custom_providers": [
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
],
})
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("beans")
assert client is not None
# Should use _read_main_model() fallback
assert model == "main-model"
def test_named_custom_no_api_key_uses_fallback(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "test"},
"custom_providers": [
{"name": "local", "base_url": "http://localhost:8080/v1"},
],
})
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("local", "test")
assert client is not None
# no-key-required should be used
def test_nonexistent_named_custom_falls_through(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "test"},
"custom_providers": [
{"name": "beans", "base_url": "http://beans.local/v1"},
],
})
from agent.auxiliary_client import resolve_provider_client
# "coffee" doesn't exist in custom_providers
client, model = resolve_provider_client("coffee", "test")
assert client is None
class TestResolveProviderClientModelNormalization:
"""Direct-provider auxiliary routing should normalize models like main runtime."""
def test_matching_native_prefix_is_stripped_for_main_provider(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "zai/glm-5.1", "provider": "zai"},
})
with (
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
"api_key": "glm-key",
"base_url": "https://api.z.ai/api/paas/v4",
}),
patch("agent.auxiliary_client.OpenAI") as mock_openai,
):
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("main", "zai/glm-5.1")
assert client is not None
assert model == "glm-5.1"
def test_non_matching_prefix_is_preserved_for_direct_provider(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "zai/glm-5.1", "provider": "zai"},
})
with (
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
"api_key": "glm-key",
"base_url": "https://api.z.ai/api/paas/v4",
}),
patch("agent.auxiliary_client.OpenAI") as mock_openai,
):
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("zai", "google/gemini-2.5-pro")
assert client is not None
assert model == "google/gemini-2.5-pro"
def test_aggregator_vendor_slug_is_preserved(self, monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client(
"openrouter", "anthropic/claude-sonnet-4.6"
)
assert client is not None
assert model == "anthropic/claude-sonnet-4.6"
class TestResolveVisionProviderClientModelNormalization:
"""Vision auto-routing should reuse the same provider-specific normalization."""
def test_vision_auto_strips_matching_main_provider_prefix(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "zai/glm-5.1", "provider": "zai"},
})
with (
patch("agent.auxiliary_client._read_nous_auth", return_value=None),
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
"api_key": "glm-key",
"base_url": "https://api.z.ai/api/paas/v4",
}),
patch("agent.auxiliary_client.OpenAI") as mock_openai,
):
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client()
assert provider == "zai"
assert client is not None
assert model == "glm-5v-turbo" # zai has dedicated vision model in _PROVIDER_VISION_MODELS
class TestVisionPathApiMode:
"""Vision path should propagate api_mode to _get_cached_client."""
def test_explicit_provider_passes_api_mode(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "test-model"},
"auxiliary": {"vision": {"api_mode": "chat_completions"}},
})
with patch("agent.auxiliary_client._get_cached_client") as mock_gcc:
mock_gcc.return_value = (MagicMock(), "test-model")
from agent.auxiliary_client import resolve_vision_provider_client
provider, client, model = resolve_vision_provider_client(provider="deepseek")
mock_gcc.assert_called_once()
_, kwargs = mock_gcc.call_args
assert kwargs.get("api_mode") == "chat_completions"
class TestProvidersDictApiModeAnthropicMessages:
"""Regression guard for #15033.
Named providers declared under the ``providers:`` dict with
``api_mode: anthropic_messages`` must route auxiliary calls through
the Anthropic Messages API (via AnthropicAuxiliaryClient), not
through an OpenAI chat-completions client.
The bug had two halves: the providers-dict branch of
``_get_named_custom_provider`` dropped the ``api_mode`` field, and
``resolve_provider_client``'s named-custom branch never read it.
"""
def test_providers_dict_propagates_api_mode(self, tmp_path, monkeypatch):
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
_write_config(tmp_path, {
"providers": {
"myrelay": {
"name": "myrelay",
"base_url": "https://example-relay.test/anthropic",
"key_env": "MYRELAY_API_KEY",
"api_mode": "anthropic_messages",
"default_model": "claude-opus-4-7",
},
},
})
from hermes_cli.runtime_provider import _get_named_custom_provider
entry = _get_named_custom_provider("myrelay")
assert entry is not None
assert entry.get("api_mode") == "anthropic_messages"
assert entry.get("base_url") == "https://example-relay.test/anthropic"
assert entry.get("api_key") == "sk-test"
def test_providers_dict_invalid_api_mode_is_dropped(self, tmp_path):
_write_config(tmp_path, {
"providers": {
"weird": {
"name": "weird",
"base_url": "https://example.test",
"api_mode": "bogus_nonsense",
"default_model": "x",
},
},
})
from hermes_cli.runtime_provider import _get_named_custom_provider
entry = _get_named_custom_provider("weird")
assert entry is not None
assert "api_mode" not in entry
def test_providers_dict_without_api_mode_is_unchanged(self, tmp_path):
_write_config(tmp_path, {
"providers": {
"localchat": {
"name": "localchat",
"base_url": "http://127.0.0.1:1234/v1",
"api_key": "local-key",
"default_model": "llama-3",
},
},
})
from hermes_cli.runtime_provider import _get_named_custom_provider
entry = _get_named_custom_provider("localchat")
assert entry is not None
assert "api_mode" not in entry
def test_resolve_provider_client_returns_anthropic_client(self, tmp_path, monkeypatch):
"""Named custom provider with api_mode=anthropic_messages must
route through AnthropicAuxiliaryClient."""
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
_write_config(tmp_path, {
"providers": {
"myrelay": {
"name": "myrelay",
"base_url": "https://example-relay.test/anthropic",
"key_env": "MYRELAY_API_KEY",
"api_mode": "anthropic_messages",
"default_model": "claude-opus-4-7",
},
},
})
from agent.auxiliary_client import (
resolve_provider_client,
AnthropicAuxiliaryClient,
AsyncAnthropicAuxiliaryClient,
)
sync_client, sync_model = resolve_provider_client("myrelay", async_mode=False)
assert isinstance(sync_client, AnthropicAuxiliaryClient), (
f"expected AnthropicAuxiliaryClient, got {type(sync_client).__name__}"
)
assert sync_model == "claude-opus-4-7"
async_client, async_model = resolve_provider_client("myrelay", async_mode=True)
assert isinstance(async_client, AsyncAnthropicAuxiliaryClient), (
f"expected AsyncAnthropicAuxiliaryClient, got {type(async_client).__name__}"
)
assert async_model == "claude-opus-4-7"
def test_aux_task_override_routes_named_provider_to_anthropic(self, tmp_path, monkeypatch):
"""The full chain: auxiliary.<task>.provider: myrelay with
api_mode anthropic_messages must produce an Anthropic client."""
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
_write_config(tmp_path, {
"providers": {
"myrelay": {
"name": "myrelay",
"base_url": "https://example-relay.test/anthropic",
"key_env": "MYRELAY_API_KEY",
"api_mode": "anthropic_messages",
"default_model": "claude-opus-4-7",
},
},
"auxiliary": {
"compression": {
"provider": "myrelay",
"model": "claude-sonnet-4.6",
},
},
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
})
from agent.auxiliary_client import (
get_async_text_auxiliary_client,
get_text_auxiliary_client,
AnthropicAuxiliaryClient,
AsyncAnthropicAuxiliaryClient,
)
async_client, async_model = get_async_text_auxiliary_client("compression")
assert isinstance(async_client, AsyncAnthropicAuxiliaryClient)
assert async_model == "claude-sonnet-4.6"
sync_client, sync_model = get_text_auxiliary_client("compression")
assert isinstance(sync_client, AnthropicAuxiliaryClient)
assert sync_model == "claude-sonnet-4.6"
def test_provider_without_api_mode_still_uses_openai(self, tmp_path):
"""Named providers that don't declare api_mode should still go
through the plain OpenAI-wire path (no regression)."""
_write_config(tmp_path, {
"providers": {
"localchat": {
"name": "localchat",
"base_url": "http://127.0.0.1:1234/v1",
"api_key": "local-key",
"default_model": "llama-3",
},
},
})
from agent.auxiliary_client import resolve_provider_client
from openai import OpenAI, AsyncOpenAI
sync_client, _ = resolve_provider_client("localchat", async_mode=False)
# sync returns the raw OpenAI client
assert isinstance(sync_client, OpenAI)
async_client, _ = resolve_provider_client("localchat", async_mode=True)
assert isinstance(async_client, AsyncOpenAI)
class TestCustomProviderAliasCollision:
"""A user-declared custom_providers entry whose name matches a built-in
*alias* (not a canonical provider) must win over the built-in.
Regression guard for #15743: users who defined fallback_model pointing at
a custom_providers entry named ``kimi`` were having requests routed to
the built-in kimi-coding endpoint because ``_normalize_aux_provider``
rewrote ``kimi`` → ``kimi-coding`` before the named-custom lookup.
"""
def test_custom_named_kimi_wins_over_builtin_alias(self, tmp_path):
_write_config(tmp_path, {
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
"custom_providers": [
{
"name": "kimi",
"base_url": "https://my-custom-kimi.example.com/v1",
"api_key": "my-kimi-key",
"models": {"my-kimi-model": {"context_length": 200000}},
},
],
})
from agent.auxiliary_client import resolve_provider_client
from openai import OpenAI
client, model = resolve_provider_client("kimi", model="my-kimi-model", raw_codex=True)
assert isinstance(client, OpenAI)
assert "my-custom-kimi.example.com" in str(client.base_url)
assert client.api_key == "my-kimi-key"
assert model == "my-kimi-model"
def test_bare_kimi_without_custom_still_routes_to_builtin(self, tmp_path, monkeypatch):
"""Regression guard: bare 'kimi' with no custom entry must still
reach the built-in kimi-coding provider."""
_write_config(tmp_path, {
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
})
monkeypatch.setenv("KIMI_API_KEY", "builtin-kimi-key")
from agent.auxiliary_client import resolve_provider_client
client, _ = resolve_provider_client("kimi", model="kimi-k2-0905-preview", raw_codex=True)
assert client is not None
base_url = str(client.base_url)
# Built-in kimi-coding points at api.moonshot.ai
assert "moonshot" in base_url or "kimi" in base_url, f"unexpected base_url {base_url!r}"
def test_explicit_overrides_applied_on_api_key_branch(self, tmp_path, monkeypatch):
"""Explicit base_url/api_key from the caller must override the
registered provider's defaults on the API-key branch. Used by
_try_activate_fallback to route a fallback through a built-in
provider name but targeting a user-supplied endpoint."""
_write_config(tmp_path, {
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
})
monkeypatch.setenv("KIMI_API_KEY", "builtin-kimi-key")
from agent.auxiliary_client import resolve_provider_client
from openai import OpenAI
client, _ = resolve_provider_client(
"kimi-coding", model="kimi-k2", raw_codex=True,
explicit_base_url="https://override.example.com",
explicit_api_key="override-key",
)
assert isinstance(client, OpenAI)
assert "override.example.com" in str(client.base_url)
assert client.api_key == "override-key"
@@ -0,0 +1,237 @@
"""Tests for transport auto-detection in agent.auxiliary_client.
Auxiliary clients must pick the correct wire protocol (OpenAI
chat.completions vs native Anthropic Messages) based on the endpoint,
regardless of which resolve_provider_client branch built them.
Regression target (April 2026): Kimi Coding Plan's ``api.kimi.com/coding``
endpoint only speaks Anthropic Messages — sending ``kimi-for-coding`` over
chat.completions returns 404 "resource_not_found_error". The named
``kimi-coding`` provider branch in resolve_provider_client used to build a
plain OpenAI client, so title generation / vision / compression /
web_extract all failed on Kimi Coding Plan users.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch):
for key in (
"OPENAI_API_KEY", "OPENAI_BASE_URL",
"ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN",
"KIMI_API_KEY", "KIMI_CODING_API_KEY", "KIMI_BASE_URL",
):
monkeypatch.delenv(key, raising=False)
# ---------------------------------------------------------------------------
# URL detection helper
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("url,expected,label", [
("https://api.kimi.com/coding/v1", True, "Kimi Coding Plan /v1"),
("https://api.kimi.com/coding", True, "Kimi Coding Plan no /v1"),
("https://api.moonshot.ai/v1", False, "Moonshot legacy"),
("https://api.minimax.io/anthropic", True, "MiniMax /anthropic"),
("https://litellm.example.com/v1/anthropic", True, "/anthropic suffix"),
("https://api.anthropic.com", True, "native Anthropic"),
("https://api.anthropic.com/v1", True, "native Anthropic /v1"),
("https://openrouter.ai/api/v1", False, "OpenRouter"),
("https://api.openai.com/v1", False, "OpenAI"),
("https://inference-api.nousresearch.com/v1", False, "Nous"),
("", False, "empty"),
(None, False, "None"),
])
def test_endpoint_speaks_anthropic_messages(url, expected, label):
from agent.auxiliary_client import _endpoint_speaks_anthropic_messages
assert _endpoint_speaks_anthropic_messages(url) is expected, (
f"{label}: {url!r} should be {expected}"
)
# ---------------------------------------------------------------------------
# _maybe_wrap_anthropic decision table
# ---------------------------------------------------------------------------
def test_maybe_wrap_anthropic_rewraps_kimi_coding_url():
"""Plain OpenAI client pointed at api.kimi.com/coding gets rewrapped."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
plain_client = MagicMock(name="plain_openai")
fake_anthropic = MagicMock(name="anthropic_sdk_client")
with patch(
"agent.anthropic_adapter.build_anthropic_client",
return_value=fake_anthropic,
):
result = _maybe_wrap_anthropic(
plain_client, "kimi-for-coding", "sk-kimi-test",
"https://api.kimi.com/coding", api_mode=None,
)
assert isinstance(result, AnthropicAuxiliaryClient)
def test_maybe_wrap_anthropic_rewraps_slash_anthropic_url():
"""Plain OpenAI client pointed at any /anthropic URL gets rewrapped."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
plain_client = MagicMock(name="plain_openai")
fake_anthropic = MagicMock(name="anthropic_sdk_client")
with patch(
"agent.anthropic_adapter.build_anthropic_client",
return_value=fake_anthropic,
):
result = _maybe_wrap_anthropic(
plain_client, "MiniMax-M2.7", "mm-key",
"https://api.minimax.io/anthropic", api_mode=None,
)
assert isinstance(result, AnthropicAuxiliaryClient)
def test_maybe_wrap_anthropic_skips_openai_wire_urls():
"""OpenRouter / OpenAI / Moonshot-legacy stay as plain OpenAI clients."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
plain_client = MagicMock(name="plain_openai")
# No patch on build_anthropic_client — if the function tried to call it,
# we'd get an AttributeError-style failure. The point is it shouldn't.
result = _maybe_wrap_anthropic(
plain_client, "claude-sonnet-4.6", "sk-or-test",
"https://openrouter.ai/api/v1", api_mode=None,
)
assert result is plain_client
assert not isinstance(result, AnthropicAuxiliaryClient)
def test_maybe_wrap_anthropic_respects_explicit_chat_completions():
"""api_mode=chat_completions overrides URL heuristics."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
plain_client = MagicMock(name="plain_openai")
result = _maybe_wrap_anthropic(
plain_client, "kimi-for-coding", "sk-kimi-test",
"https://api.kimi.com/coding",
api_mode="chat_completions", # explicit override
)
assert result is plain_client, "Explicit chat_completions must bypass wrap"
assert not isinstance(result, AnthropicAuxiliaryClient)
def test_maybe_wrap_anthropic_honors_explicit_anthropic_messages():
"""api_mode=anthropic_messages wraps even when URL wouldn't trigger."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
plain_client = MagicMock(name="plain_openai")
fake_anthropic = MagicMock(name="anthropic_sdk_client")
with patch(
"agent.anthropic_adapter.build_anthropic_client",
return_value=fake_anthropic,
):
result = _maybe_wrap_anthropic(
plain_client, "model-name", "some-key",
"https://opaque.internal/v1", # URL alone wouldn't trigger
api_mode="anthropic_messages",
)
assert isinstance(result, AnthropicAuxiliaryClient)
def test_maybe_wrap_anthropic_double_wrap_safe():
"""Already-wrapped AnthropicAuxiliaryClient passes through unchanged."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
already_wrapped = MagicMock(spec=AnthropicAuxiliaryClient)
result = _maybe_wrap_anthropic(
already_wrapped, "model", "key",
"https://api.kimi.com/coding", api_mode=None,
)
assert result is already_wrapped
def test_maybe_wrap_anthropic_codex_client_passes_through():
"""CodexAuxiliaryClient is never re-dispatched."""
from agent.auxiliary_client import (
_maybe_wrap_anthropic,
CodexAuxiliaryClient,
AnthropicAuxiliaryClient,
)
codex_client = MagicMock(spec=CodexAuxiliaryClient)
result = _maybe_wrap_anthropic(
codex_client, "model", "key",
"https://api.kimi.com/coding", api_mode=None,
)
assert result is codex_client
assert not isinstance(result, AnthropicAuxiliaryClient)
def test_maybe_wrap_anthropic_sdk_missing_falls_back():
"""ImportError on anthropic SDK returns plain client with warning."""
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
plain_client = MagicMock(name="plain_openai")
def _raise_import(*args, **kwargs):
raise ImportError("no anthropic SDK")
with patch(
"agent.anthropic_adapter.build_anthropic_client",
side_effect=_raise_import,
):
# The ImportError is caught on the `from ... import` line inside
# _maybe_wrap_anthropic, which runs before build_anthropic_client is
# called. To exercise the ImportError path we need to patch the
# module lookup itself.
import sys as _sys
saved = _sys.modules.get("agent.anthropic_adapter")
_sys.modules["agent.anthropic_adapter"] = None # force ImportError
try:
result = _maybe_wrap_anthropic(
plain_client, "kimi-for-coding", "sk-kimi-test",
"https://api.kimi.com/coding", api_mode=None,
)
finally:
if saved is not None:
_sys.modules["agent.anthropic_adapter"] = saved
else:
_sys.modules.pop("agent.anthropic_adapter", None)
assert result is plain_client
assert not isinstance(result, AnthropicAuxiliaryClient)
# ---------------------------------------------------------------------------
# Integration: resolve_provider_client for named kimi-coding provider
# ---------------------------------------------------------------------------
def test_resolve_provider_client_kimi_coding_wraps_anthropic(monkeypatch, tmp_path):
"""End-to-end: resolve_provider_client('kimi-coding', 'kimi-for-coding')
must return AnthropicAuxiliaryClient because /coding speaks Anthropic.
This is the primary regression guard: the bug that caused title
generation 404s on every Kimi Coding Plan user after the "main model
for every user" aux design shipped.
"""
from agent.auxiliary_client import (
resolve_provider_client,
AnthropicAuxiliaryClient,
)
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
# sk-kimi- prefix triggers /coding endpoint auto-detection
monkeypatch.setenv("KIMI_API_KEY", "sk-kimi-faketesttoken123")
client, model = resolve_provider_client("kimi-coding", "kimi-for-coding")
assert client is not None, "Should resolve a client"
assert isinstance(client, AnthropicAuxiliaryClient), (
"Kimi Coding Plan endpoint (api.kimi.com/coding) speaks Anthropic "
"Messages — aux client MUST be AnthropicAuxiliaryClient, got "
f"{type(client).__name__}"
)
assert "kimi.com/coding" in str(client.base_url)
+661
View File
@@ -0,0 +1,661 @@
"""Tests for the Microsoft Entra ID adapter (agent/azure_identity_adapter.py).
Covers:
- Scope resolution per Azure host shape
- Display masking for callable + string + None inputs
- Cache-fingerprint stability under callable refresh
- is_token_provider truthiness on callables vs strings
- EntraIdentityConfig serialization round-trip
- Token provider construction with mocked azure-identity
- Credential cache reuse + reset
- has_azure_identity_credentials timeout / failure paths
- describe_active_credential structural reporting
- Lazy-install error path when azure-identity absent + lazy installs
disabled
We mock azure.identity at the import boundary rather than hitting any
real Azure endpoint. Tests must remain hermetic per AGENTS.md.
"""
from __future__ import annotations
import sys
from collections.abc import Callable
from types import SimpleNamespace
from typing import cast
import pytest
# Ensure we always import a fresh adapter module — credential caches in
# the adapter persist across tests otherwise, polluting assertions
# about cache invalidation.
@pytest.fixture(autouse=True)
def _reset_adapter_cache():
from agent.azure_identity_adapter import reset_credential_cache
reset_credential_cache()
yield
reset_credential_cache()
# ---------------------------------------------------------------------------
# Scope constant
# ---------------------------------------------------------------------------
class TestEntraScopeConstant:
"""Pin the Microsoft-documented Foundry inference scope.
Microsoft's official samples for both ``*.openai.azure.com`` and
``*.services.ai.azure.com`` use ``https://ai.azure.com/.default``.
The older ``cognitiveservices.azure.com/.default`` is the
control-plane scope and is rejected for inference by newer
Azure OpenAI / Foundry resources.
Users with sovereign-cloud or unusual-tenant requirements pass the
scope explicitly via ``model.entra.scope`` in ``config.yaml``.
Refs:
* https://learn.microsoft.com/azure/ai-foundry/openai/how-to/managed-identity
* https://learn.microsoft.com/azure/ai-foundry/foundry-models/how-to/configure-entra-id
"""
def test_default_scope_matches_microsoft_documentation(self):
from agent.azure_identity_adapter import SCOPE_AI_AZURE_DEFAULT
assert SCOPE_AI_AZURE_DEFAULT == "https://ai.azure.com/.default"
# ---------------------------------------------------------------------------
# Cache fingerprint + http-bearer helpers
# ---------------------------------------------------------------------------
class TestMaterializeBearerForHttp:
"""The only helper that mints a real bearer JWT — must call the
callable exactly once and never fall through to display masking."""
def test_callable_is_invoked_and_returns_token(self):
from agent.azure_identity_adapter import materialize_bearer_for_http
invoked = {"count": 0}
def provider():
invoked["count"] += 1
return "fresh-jwt"
assert materialize_bearer_for_http(provider) == "fresh-jwt"
assert invoked["count"] == 1
def test_string_passes_through(self):
from agent.azure_identity_adapter import materialize_bearer_for_http
assert materialize_bearer_for_http("plain-key") == "plain-key"
def test_callable_returning_empty_raises(self):
from agent.azure_identity_adapter import materialize_bearer_for_http
with pytest.raises(ValueError):
materialize_bearer_for_http(lambda: "")
def test_empty_string_raises(self):
from agent.azure_identity_adapter import materialize_bearer_for_http
with pytest.raises(ValueError):
materialize_bearer_for_http("")
with pytest.raises(ValueError):
materialize_bearer_for_http(None)
# ---------------------------------------------------------------------------
# build_bearer_http_client — the Anthropic-on-Foundry bridge
# ---------------------------------------------------------------------------
class TestBuildBearerHttpClient:
"""``build_bearer_http_client`` returns an ``httpx.Client`` whose
request event hook mints a fresh JWT per outbound request. This is
how Entra ID auth reaches the Anthropic SDK (which does not accept
callable ``auth_token``)."""
def test_returns_httpx_client_with_request_hook(self):
import httpx
from agent.azure_identity_adapter import build_bearer_http_client
client = build_bearer_http_client(lambda: "jwt")
try:
assert isinstance(client, httpx.Client)
hooks = client.event_hooks.get("request", [])
assert len(hooks) >= 1
finally:
client.close()
def test_hook_overrides_authorization_header(self):
import httpx
from agent.azure_identity_adapter import build_bearer_http_client
minted_tokens = []
def provider():
minted_tokens.append(f"jwt-{len(minted_tokens) + 1}")
return minted_tokens[-1]
client = build_bearer_http_client(provider)
try:
hook = client.event_hooks["request"][0]
# Build a request with conflicting pre-set headers and verify
# the hook strips them and installs the fresh bearer.
req = httpx.Request(
"POST", "https://example.com/v1/messages",
headers={
"Authorization": "Bearer stale-token",
"api-key": "static-key",
"x-api-key": "static-key",
},
json={"hello": "world"},
)
hook(req)
assert req.headers["Authorization"] == "Bearer jwt-1"
# The static-key headers must be stripped — sending both
# auth values would be ambiguous on Azure.
assert "api-key" not in req.headers
assert "x-api-key" not in req.headers
# Second invocation mints a fresh token.
req2 = httpx.Request("GET", "https://example.com/v1/models")
hook(req2)
assert req2.headers["Authorization"] == "Bearer jwt-2"
assert len(minted_tokens) == 2
finally:
client.close()
def test_hook_strips_auth_headers_and_warns_when_token_provider_fails(self, caplog):
"""When the token provider fails (chain exhausted, IMDS down, az
login expired), the hook must:
1. Log at WARNING level so the misconfiguration is visible at
default log level (not buried at DEBUG).
2. Strip any pre-set Authorization headers — including the
placeholder ``entra-id-bearer-via-http-hook`` sentinel that
:func:`_build_anthropic_client_with_bearer_hook` sets on the
Anthropic SDK constructor. This produces a clean
"missing auth" 401 from Azure rather than a sentinel-bearing
401 that's harder to diagnose AND avoids leaking the
sentinel string into upstream access logs.
"""
import logging
import httpx
from agent.azure_identity_adapter import build_bearer_http_client
def bad_provider():
return "" # empty token → materialize_bearer_for_http raises
client = build_bearer_http_client(bad_provider)
try:
hook = client.event_hooks["request"][0]
req = httpx.Request(
"POST", "https://example.com/v1/messages",
headers={
"Authorization": "Bearer entra-id-bearer-via-http-hook",
"api-key": "leaked-placeholder",
},
)
with caplog.at_level(logging.WARNING, logger="agent.azure_identity_adapter"):
hook(req) # Must not raise.
# Pre-set auth headers stripped — no sentinel makes it to Azure.
assert "Authorization" not in req.headers
assert "api-key" not in req.headers
# WARNING was logged so the user sees the misconfiguration.
assert any(
rec.levelno == logging.WARNING and "Entra ID token provider" in rec.message
for rec in caplog.records
)
finally:
client.close()
def test_rejects_non_callable_provider(self):
from agent.azure_identity_adapter import build_bearer_http_client
with pytest.raises(ValueError):
build_bearer_http_client(cast(Callable[[], str], "plain-string-not-callable"))
with pytest.raises(ValueError):
build_bearer_http_client(cast(Callable[[], str], None))
def test_forwards_httpx_kwargs(self):
import httpx
from agent.azure_identity_adapter import build_bearer_http_client
timeout = httpx.Timeout(60.0, connect=5.0)
client = build_bearer_http_client(lambda: "jwt", timeout=timeout)
try:
# httpx stores the timeout per-pool; just sanity-check it was
# accepted without TypeError.
assert client is not None
finally:
client.close()
class TestIsTokenProvider:
def test_callable_is_token_provider(self):
from agent.azure_identity_adapter import is_token_provider
assert is_token_provider(lambda: "x") is True
def test_string_is_not_token_provider(self):
from agent.azure_identity_adapter import is_token_provider
assert is_token_provider("static-key") is False
# ``str`` instances are technically callable in some edge cases
# — confirm they're never classified as token providers.
assert is_token_provider("") is False
# ---------------------------------------------------------------------------
# EntraIdentityConfig
# ---------------------------------------------------------------------------
class TestEntraIdentityConfig:
"""The serializable config that crosses multiprocessing boundaries —
must round-trip through dict cleanly and never lose fields."""
def test_to_dict_round_trip(self):
from agent.azure_identity_adapter import EntraIdentityConfig
cfg = EntraIdentityConfig(
scope="https://ai.azure.com/.default",
exclude_interactive_browser=False,
)
rebuilt = EntraIdentityConfig.from_dict(cfg.to_dict())
assert rebuilt == cfg
def test_from_dict_handles_empty_strings(self):
from agent.azure_identity_adapter import EntraIdentityConfig
cfg = EntraIdentityConfig.from_dict({
"scope": "",
"client_id": None,
})
# Empty scope falls back to default
assert cfg.scope.endswith("/.default")
def test_from_dict_ignores_legacy_identity_keys(self):
"""Old config.yaml that still has model.entra.client_id /
tenant_id / authority should not crash from_dict — those values
are now read from AZURE_* env vars by azure-identity directly."""
from agent.azure_identity_adapter import EntraIdentityConfig
cfg = EntraIdentityConfig.from_dict({
"tenant_id": "legacy-tenant",
"authority": "https://login.partner.microsoftonline.cn",
"client_id": "user-mi-client",
})
# Legacy keys silently ignored — no crash, no surprise field on the dataclass.
assert not hasattr(cfg, "client_id")
assert not hasattr(cfg, "tenant_id")
assert not hasattr(cfg, "authority")
def test_constructor_normalizes_empty_scope(self):
from agent.azure_identity_adapter import EntraIdentityConfig
cfg = EntraIdentityConfig(scope="")
assert cfg.scope.endswith("/.default")
def test_from_dict_default_scope_override(self):
from agent.azure_identity_adapter import EntraIdentityConfig
cfg = EntraIdentityConfig.from_dict(
{"scope": ""},
default_scope="https://custom.example/.default",
)
assert cfg.scope == "https://custom.example/.default"
def test_dataclass_is_frozen(self):
# Frozen dataclasses are hashable / safe to pass through caches.
from agent.azure_identity_adapter import EntraIdentityConfig
cfg = EntraIdentityConfig()
with pytest.raises((AttributeError, Exception)):
setattr(cfg, "scope", "mutated")
# ---------------------------------------------------------------------------
# Credential / token provider construction
# ---------------------------------------------------------------------------
class _FakeAzureIdentity:
"""Stand-in for the ``azure.identity`` module.
Captures kwargs passed to ``DefaultAzureCredential`` so tests can
assert how config flows into the SDK.
"""
def __init__(self):
self.last_credential_kwargs = None
self.last_scope = None
self.credential_count = 0
def DefaultAzureCredential(self, **kwargs): # noqa: N802 — match SDK
self.last_credential_kwargs = kwargs
self.credential_count += 1
return SimpleNamespace(
get_token=lambda scope: SimpleNamespace(token="fake-jwt", expires_on=9999999999),
kwargs=kwargs,
)
def get_bearer_token_provider(self, credential, scope):
self.last_scope = scope
# Return a callable that mints a token when invoked.
return lambda: f"jwt-for-{scope}"
@pytest.fixture
def fake_azure_identity(monkeypatch):
"""Install a fake azure.identity into sys.modules and stub the
adapter's `_require_azure_identity` so all tests use the fake."""
fake = _FakeAzureIdentity()
fake_module = SimpleNamespace(
DefaultAzureCredential=fake.DefaultAzureCredential,
get_bearer_token_provider=fake.get_bearer_token_provider,
)
monkeypatch.setitem(sys.modules, "azure", SimpleNamespace(identity=fake_module))
monkeypatch.setitem(sys.modules, "azure.identity", fake_module)
# The adapter's `_require_azure_identity` does its own import, so
# patch that too to make sure tests never hit the real package's
# singleton state.
from agent import azure_identity_adapter as _adapter
monkeypatch.setattr(_adapter, "_require_azure_identity", lambda: fake_module)
return fake
class TestBuildCredential:
def test_default_kwargs_are_minimal(self, fake_azure_identity):
"""SDK default for ``exclude_interactive_browser_credential`` is
True; we only pass it when the user opts IN to interactive
browser auth. Tenant / authority / service principal config
flow through the standard ``AZURE_*`` env vars (read by
azure-identity directly), not Hermes config kwargs."""
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
cred = build_credential(EntraIdentityConfig())
kwargs = fake_azure_identity.last_credential_kwargs
# Default config should produce empty kwargs — SDK uses its own
# defaults plus env-var-driven settings.
assert kwargs == {}
assert cred is not None
def test_interactive_browser_opt_in(self, fake_azure_identity):
"""When the user explicitly sets
``exclude_interactive_browser=False``, the SDK kwarg is set to
False. Without the opt-in we don't pass the kwarg at all (SDK
default is True / browser excluded)."""
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
build_credential(EntraIdentityConfig(exclude_interactive_browser=False))
kwargs = fake_azure_identity.last_credential_kwargs
assert kwargs["exclude_interactive_browser_credential"] is False
def test_credential_is_cached_per_config(self, fake_azure_identity):
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
cfg = EntraIdentityConfig(scope="s1")
c1 = build_credential(cfg)
c2 = build_credential(cfg)
assert c1 is c2
assert fake_azure_identity.credential_count == 1
def test_distinct_configs_get_distinct_credentials(self, fake_azure_identity):
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
c1 = build_credential(EntraIdentityConfig(scope="s1"))
c2 = build_credential(EntraIdentityConfig(scope="s2"))
assert c1 is not c2
assert fake_azure_identity.credential_count == 2
def test_reset_cache_invalidates(self, fake_azure_identity):
from agent.azure_identity_adapter import (
EntraIdentityConfig,
build_credential,
reset_credential_cache,
)
cfg = EntraIdentityConfig(scope="x")
c1 = build_credential(cfg)
reset_credential_cache()
c2 = build_credential(cfg)
assert c1 is not c2
class TestBuildTokenProvider:
def test_returns_callable_for_scope(self, fake_azure_identity):
from agent.azure_identity_adapter import build_token_provider
provider = build_token_provider(scope="https://ai.azure.com/.default")
assert callable(provider)
assert provider() == "jwt-for-https://ai.azure.com/.default"
assert fake_azure_identity.last_scope == "https://ai.azure.com/.default"
def test_falls_back_to_default_scope_when_unspecified(self, fake_azure_identity):
"""When neither ``scope`` nor ``config`` is provided,
``build_token_provider`` uses ``SCOPE_AI_AZURE_DEFAULT`` —
Microsoft's documented Foundry inference scope. ``base_url`` is
accepted for back-compat but ignored."""
from agent.azure_identity_adapter import (
SCOPE_AI_AZURE_DEFAULT,
build_token_provider,
)
build_token_provider(base_url="https://r.openai.azure.com/openai/v1")
assert fake_azure_identity.last_scope == SCOPE_AI_AZURE_DEFAULT
def test_explicit_scope_wins_over_base_url(self, fake_azure_identity):
from agent.azure_identity_adapter import build_token_provider
build_token_provider(
scope="https://override.example/.default",
base_url="https://r.openai.azure.com/openai/v1",
)
assert fake_azure_identity.last_scope == "https://override.example/.default"
def test_config_object_wins_over_kwargs(self, fake_azure_identity):
from agent.azure_identity_adapter import (
EntraIdentityConfig,
build_token_provider,
)
cfg = EntraIdentityConfig(scope="cfg-scope")
build_token_provider(scope="ignored", config=cfg)
assert fake_azure_identity.last_scope == "cfg-scope"
assert fake_azure_identity.last_credential_kwargs == {}
# ---------------------------------------------------------------------------
# Lazy-install / missing-package surface
# ---------------------------------------------------------------------------
class TestRequireAzureIdentityMissing:
def test_clear_error_when_lazy_install_disabled(self, monkeypatch):
"""When azure-identity isn't importable AND lazy installs are
off, the adapter must raise ImportError with an actionable
message, not propagate FeatureUnavailable."""
from agent import azure_identity_adapter as _adapter
# Force the import path to fail.
original_import = __builtins__["__import__"] if isinstance(__builtins__, dict) else __import__
def _fake_import(name, *args, **kwargs):
if name == "azure.identity" or name.startswith("azure.identity."):
raise ImportError("simulated missing azure-identity")
return original_import(name, *args, **kwargs)
monkeypatch.setattr("builtins.__import__", _fake_import)
# Simulate lazy installs disabled.
from tools.lazy_deps import FeatureUnavailable
def _fake_ensure(*args, **kwargs):
raise FeatureUnavailable(
"provider.azure_identity",
("azure-identity==1.25.3",),
"lazy installs disabled (test simulation)",
)
# The adapter calls ``ensure`` from ``tools.lazy_deps``; intercept
# it by patching the actual symbol path.
monkeypatch.setattr("tools.lazy_deps.ensure", _fake_ensure)
with pytest.raises(ImportError) as exc_info:
_adapter._require_azure_identity()
msg = str(exc_info.value)
assert "azure-identity" in msg
assert "Foundry" in msg or "foundry" in msg.lower()
# ---------------------------------------------------------------------------
# has_azure_identity_credentials probe (timeout-bounded)
# ---------------------------------------------------------------------------
class TestHasAzureIdentityCredentials:
def test_returns_false_when_package_missing_and_install_disabled(self, monkeypatch):
from agent import azure_identity_adapter as _adapter
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
assert _adapter.has_azure_identity_credentials(
"https://x/.default", allow_install=False,
) is False
def test_lazy_install_triggered_when_package_missing(self, monkeypatch):
"""With allow_install=True (default), the probe must trigger the
lazy-install path before bailing — otherwise the wizard's
``preflight`` would silently fail for fresh installs that haven't
run ``pip install azure-identity`` yet."""
from agent import azure_identity_adapter as _adapter
installed = {"called": False}
def _fake_install():
installed["called"] = True
# After install, pretend the package is now importable.
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
return SimpleNamespace(
DefaultAzureCredential=lambda **kw: SimpleNamespace(
kwargs=kw,
get_token=lambda scope: SimpleNamespace(token="post-install-jwt", expires_on=0),
),
get_bearer_token_provider=lambda c, s: lambda: "x",
)
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
monkeypatch.setattr(_adapter, "_require_azure_identity", _fake_install)
# Provide a credential factory so the probe proceeds after install.
monkeypatch.setattr(
_adapter, "build_credential",
lambda config: SimpleNamespace(
get_token=lambda scope: SimpleNamespace(token="probe-jwt", expires_on=0),
),
)
result = _adapter.has_azure_identity_credentials(
"https://x/.default", timeout_seconds=0.5,
)
assert installed["called"] is True, (
"has_azure_identity_credentials must trigger lazy install "
"before bailing"
)
assert result is True
def test_returns_true_on_successful_token_mint(self, fake_azure_identity):
from agent.azure_identity_adapter import has_azure_identity_credentials
assert has_azure_identity_credentials("https://x/.default", timeout_seconds=0.5) is True
def test_returns_false_when_get_token_raises(self, monkeypatch):
from agent import azure_identity_adapter as _adapter
def _failing_credential(_config):
class _Cred:
def get_token(self, scope):
raise RuntimeError("simulated chain exhaustion")
return _Cred()
monkeypatch.setattr(_adapter, "build_credential", _failing_credential)
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
assert _adapter.has_azure_identity_credentials("https://x/.default", timeout_seconds=0.5) is False
def test_returns_false_on_timeout(self, monkeypatch):
"""Slow IMDS / network must time out, not hang the caller."""
import threading
from agent import azure_identity_adapter as _adapter
slow_release = threading.Event()
def _slow_credential(_config):
class _Cred:
def get_token(self, scope):
# Block forever from the test's perspective; the
# adapter must give up via its thread-bounded probe.
slow_release.wait(timeout=10)
return SimpleNamespace(token="never-returned", expires_on=0)
return _Cred()
monkeypatch.setattr(_adapter, "build_credential", _slow_credential)
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
try:
assert _adapter.has_azure_identity_credentials(
"https://x/.default", timeout_seconds=0.1
) is False
finally:
slow_release.set()
# ---------------------------------------------------------------------------
# describe_active_credential — used by hermes doctor + hermes auth
# ---------------------------------------------------------------------------
class TestDescribeActiveCredential:
def test_reports_not_installed(self, monkeypatch):
from agent import azure_identity_adapter as _adapter
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
info = _adapter.describe_active_credential(
scope="https://x/.default", allow_install=False,
)
assert info["ok"] is False
assert "not installed" in info["error"].lower()
assert "pip install" in info["hint"].lower()
def test_reports_install_failure(self, monkeypatch):
"""When lazy install is allowed but fails (e.g. lazy installs
disabled), the diagnostic surfaces the failure as the error."""
from agent import azure_identity_adapter as _adapter
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
def _fail_install():
raise ImportError("simulated: lazy installs disabled")
monkeypatch.setattr(_adapter, "_require_azure_identity", _fail_install)
info = _adapter.describe_active_credential(
scope="https://x/.default", allow_install=True,
)
assert info["ok"] is False
assert "lazy installs disabled" in info["error"]
assert "lazy" in info["hint"].lower()
def test_reports_env_sources_for_managed_identity(self, fake_azure_identity, monkeypatch):
from agent.azure_identity_adapter import describe_active_credential
monkeypatch.setenv("IDENTITY_ENDPOINT", "http://169.254.169.254")
info = describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
assert info["ok"] is True
sources = info.get("env_sources") or []
assert any("ManagedIdentity" in s for s in sources)
def test_reports_env_sources_for_workload_identity(self, fake_azure_identity, monkeypatch):
from agent.azure_identity_adapter import describe_active_credential
monkeypatch.setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/secrets/azure/federated-token")
info = describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
sources = info.get("env_sources") or []
assert any("WorkloadIdentity" in s for s in sources)
def test_reports_env_sources_for_service_principal(self, fake_azure_identity, monkeypatch):
from agent.azure_identity_adapter import describe_active_credential
monkeypatch.setenv("AZURE_TENANT_ID", "t")
monkeypatch.setenv("AZURE_CLIENT_ID", "c")
monkeypatch.setenv("AZURE_CLIENT_SECRET", "s")
info = describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
sources = info.get("env_sources") or []
assert any("EnvironmentCredential" in s for s in sources)
def test_reports_error_on_chain_failure(self, monkeypatch):
from agent import azure_identity_adapter as _adapter
def _failing_credential(_config):
class _Cred:
def get_token(self, scope):
raise RuntimeError("auth failed")
return _Cred()
monkeypatch.setattr(_adapter, "build_credential", _failing_credential)
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
info = _adapter.describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
assert info["ok"] is False
assert "auth failed" in info.get("error", "")
+63
View File
@@ -0,0 +1,63 @@
"""Tests for the 1M-context beta header on AWS Bedrock Claude models.
Claude Opus 4.6/4.7 and Sonnet 4.6 support a 1M context window, but on AWS
Bedrock (and Microsoft Foundry) that window is still gated behind the
``context-1m-2025-08-07`` beta header as of 2026-04. Without it, Bedrock
caps these models at 200K even though ``model_metadata.py`` advertises 1M.
These tests guard the invariant that the header is always emitted on the
Bedrock client path, and that it survives the MiniMax bearer-auth strip.
"""
from unittest.mock import MagicMock, patch
class TestBedrockContext1MBeta:
"""``context-1m-2025-08-07`` must reach Bedrock Claude requests."""
def test_common_betas_strips_1m_for_minimax(self):
"""MiniMax bearer-auth endpoints host their own models — strip 1M beta."""
from agent.anthropic_adapter import (
_common_betas_for_base_url,
_CONTEXT_1M_BETA,
)
for url in (
"https://api.minimax.io/anthropic",
"https://api.minimaxi.com/anthropic",
):
betas = _common_betas_for_base_url(url)
assert _CONTEXT_1M_BETA not in betas, (
f"1M beta must be stripped for MiniMax bearer endpoint {url}"
)
# Other betas still present
assert "interleaved-thinking-2025-05-14" in betas
def test_build_anthropic_bedrock_client_sends_1m_beta(self):
"""AnthropicBedrock client must carry the 1M beta in default_headers.
This is the load-bearing assertion for the reported bug:
without this header Bedrock serves Opus 4.6/4.7 with a 200K cap.
"""
import agent.anthropic_adapter as adapter
fake_sdk = MagicMock()
fake_sdk.AnthropicBedrock = MagicMock()
with patch.object(adapter, "_anthropic_sdk", fake_sdk):
adapter.build_anthropic_bedrock_client(region="us-west-2")
call_kwargs = fake_sdk.AnthropicBedrock.call_args.kwargs
assert call_kwargs["aws_region"] == "us-west-2"
default_headers = call_kwargs.get("default_headers") or {}
beta_header = default_headers.get("anthropic-beta", "")
assert "context-1m-2025-08-07" in beta_header, (
"Bedrock client must send context-1m-2025-08-07 or Opus 4.6/4.7 "
"silently caps at 200K context"
)
# Other common betas still present — no regression.
assert "interleaved-thinking-2025-05-14" in beta_header
assert "fine-grained-tool-streaming-2025-05-14" in beta_header
File diff suppressed because it is too large Load Diff
+592
View File
@@ -0,0 +1,592 @@
"""Integration tests for the AWS Bedrock provider wiring.
Verifies that the Bedrock provider is correctly registered in the
provider registry, model catalog, and runtime resolution pipeline.
These tests do NOT require AWS credentials or boto3 — all AWS calls
are mocked.
Note: Tests that import ``hermes_cli.auth`` or ``hermes_cli.runtime_provider``
require Python 3.10+ due to ``str | None`` type syntax in the import chain.
"""
from unittest.mock import MagicMock, patch
import pytest
class TestProviderRegistry:
"""Verify Bedrock is registered in PROVIDER_REGISTRY."""
def test_bedrock_in_registry(self):
from hermes_cli.auth import PROVIDER_REGISTRY
assert "bedrock" in PROVIDER_REGISTRY
def test_bedrock_auth_type_is_aws_sdk(self):
from hermes_cli.auth import PROVIDER_REGISTRY
pconfig = PROVIDER_REGISTRY["bedrock"]
assert pconfig.auth_type == "aws_sdk"
def test_bedrock_has_no_api_key_env_vars(self):
"""Bedrock uses the AWS SDK credential chain, not API keys."""
from hermes_cli.auth import PROVIDER_REGISTRY
pconfig = PROVIDER_REGISTRY["bedrock"]
assert pconfig.api_key_env_vars == ()
def test_bedrock_base_url_env_var(self):
from hermes_cli.auth import PROVIDER_REGISTRY
pconfig = PROVIDER_REGISTRY["bedrock"]
assert pconfig.base_url_env_var == "BEDROCK_BASE_URL"
class TestProviderAliases:
"""Verify Bedrock aliases resolve correctly."""
def test_aws_alias(self):
from hermes_cli.models import _PROVIDER_ALIASES
assert _PROVIDER_ALIASES.get("aws") == "bedrock"
def test_aws_bedrock_alias(self):
from hermes_cli.models import _PROVIDER_ALIASES
assert _PROVIDER_ALIASES.get("aws-bedrock") == "bedrock"
def test_amazon_bedrock_alias(self):
from hermes_cli.models import _PROVIDER_ALIASES
assert _PROVIDER_ALIASES.get("amazon-bedrock") == "bedrock"
def test_amazon_alias(self):
from hermes_cli.models import _PROVIDER_ALIASES
assert _PROVIDER_ALIASES.get("amazon") == "bedrock"
class TestProviderLabels:
"""Verify Bedrock appears in provider labels."""
def test_bedrock_label(self):
from hermes_cli.models import _PROVIDER_LABELS
assert _PROVIDER_LABELS.get("bedrock") == "AWS Bedrock"
class TestModelCatalog:
"""Verify Bedrock has a static model fallback list."""
def test_bedrock_has_curated_models(self):
from hermes_cli.models import _PROVIDER_MODELS
models = _PROVIDER_MODELS.get("bedrock", [])
assert len(models) > 0
def test_bedrock_models_include_claude(self):
from hermes_cli.models import _PROVIDER_MODELS
models = _PROVIDER_MODELS.get("bedrock", [])
claude_models = [m for m in models if "anthropic.claude" in m]
assert len(claude_models) > 0
def test_bedrock_models_include_nova(self):
from hermes_cli.models import _PROVIDER_MODELS
models = _PROVIDER_MODELS.get("bedrock", [])
nova_models = [m for m in models if "amazon.nova" in m]
assert len(nova_models) > 0
class TestResolveProvider:
"""Verify resolve_provider() handles bedrock correctly."""
def test_explicit_bedrock_resolves(self, monkeypatch):
"""When user explicitly requests 'bedrock', it should resolve."""
# bedrock is in the registry, so resolve_provider should return it
from hermes_cli.auth import resolve_provider
result = resolve_provider("bedrock")
assert result == "bedrock"
def test_aws_alias_resolves_to_bedrock(self):
from hermes_cli.auth import resolve_provider
result = resolve_provider("aws")
assert result == "bedrock"
def test_amazon_bedrock_alias_resolves(self):
from hermes_cli.auth import resolve_provider
result = resolve_provider("amazon-bedrock")
assert result == "bedrock"
def test_auto_detect_with_aws_credentials(self, monkeypatch):
"""When AWS credentials are present and no other provider is configured,
auto-detect should find bedrock."""
from hermes_cli.auth import resolve_provider
# Clear all other provider env vars
for var in ["OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY",
"ANTHROPIC_TOKEN", "GOOGLE_API_KEY", "DEEPSEEK_API_KEY"]:
monkeypatch.delenv(var, raising=False)
# Set AWS credentials
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
# Mock the auth store to have no active provider
with patch("hermes_cli.auth._load_auth_store", return_value={}):
result = resolve_provider("auto")
assert result == "bedrock"
class TestRuntimeProvider:
"""Verify resolve_runtime_provider() handles bedrock correctly."""
def test_bedrock_runtime_resolution(self, monkeypatch):
from hermes_cli.runtime_provider import resolve_runtime_provider
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
monkeypatch.setenv("AWS_REGION", "eu-west-1")
# Mock resolve_provider to return bedrock
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}):
result = resolve_runtime_provider(requested="bedrock")
assert result["provider"] == "bedrock"
assert result["api_mode"] == "bedrock_converse"
assert result["region"] == "eu-west-1"
assert "bedrock-runtime.eu-west-1.amazonaws.com" in result["base_url"]
assert result["api_key"] == "aws-sdk"
def test_bedrock_runtime_default_region(self, monkeypatch):
from hermes_cli.runtime_provider import resolve_runtime_provider
monkeypatch.setenv("AWS_PROFILE", "default")
monkeypatch.delenv("AWS_REGION", raising=False)
monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}):
result = resolve_runtime_provider(requested="bedrock")
assert result["region"] == "us-east-1"
def test_bedrock_runtime_no_credentials_raises_on_auto_detect(self, monkeypatch):
"""When bedrock is auto-detected (not explicitly requested) and no
credentials are found, runtime resolution should raise AuthError."""
from hermes_cli.runtime_provider import resolve_runtime_provider
from hermes_cli.auth import AuthError
# Clear all AWS env vars
for var in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_PROFILE",
"AWS_BEARER_TOKEN_BEDROCK", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
"AWS_WEB_IDENTITY_TOKEN_FILE"]:
monkeypatch.delenv(var, raising=False)
# Mock both the provider resolution and boto3's credential chain
mock_session = MagicMock()
mock_session.get_credentials.return_value = None
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}), \
patch("hermes_cli.runtime_provider.resolve_requested_provider", return_value="auto"), \
patch.dict("sys.modules", {"botocore": MagicMock(), "botocore.session": MagicMock()}):
import botocore.session as _bs
_bs.get_session = MagicMock(return_value=mock_session)
with pytest.raises(AuthError, match="No AWS credentials"):
resolve_runtime_provider(requested="auto")
def test_bedrock_runtime_explicit_skips_credential_check(self, monkeypatch):
"""When user explicitly requests bedrock, trust boto3's credential chain
even if env-var detection finds nothing (covers IMDS, SSO, etc.)."""
from hermes_cli.runtime_provider import resolve_runtime_provider
# No AWS env vars set — but explicit bedrock request should not raise
for var in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_PROFILE",
"AWS_BEARER_TOKEN_BEDROCK"]:
monkeypatch.delenv(var, raising=False)
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}):
result = resolve_runtime_provider(requested="bedrock")
assert result["provider"] == "bedrock"
assert result["api_mode"] == "bedrock_converse"
# ---------------------------------------------------------------------------
# providers.py integration
# ---------------------------------------------------------------------------
class TestProvidersModule:
"""Verify bedrock is wired into hermes_cli/providers.py."""
def test_bedrock_alias_in_providers(self):
from hermes_cli.providers import ALIASES
assert ALIASES.get("bedrock") is None # "bedrock" IS the canonical name, not an alias
assert ALIASES.get("aws") == "bedrock"
assert ALIASES.get("aws-bedrock") == "bedrock"
def test_bedrock_transport_mapping(self):
from hermes_cli.providers import TRANSPORT_TO_API_MODE
assert TRANSPORT_TO_API_MODE.get("bedrock_converse") == "bedrock_converse"
def test_determine_api_mode_from_bedrock_url(self):
from hermes_cli.providers import determine_api_mode
assert determine_api_mode(
"unknown", "https://bedrock-runtime.us-east-1.amazonaws.com"
) == "bedrock_converse"
def test_label_override(self):
from hermes_cli.providers import _LABEL_OVERRIDES
assert _LABEL_OVERRIDES.get("bedrock") == "AWS Bedrock"
# ---------------------------------------------------------------------------
# Error classifier integration
# ---------------------------------------------------------------------------
class TestErrorClassifierBedrock:
"""Verify Bedrock error patterns are in the global error classifier."""
def test_throttling_in_rate_limit_patterns(self):
from agent.error_classifier import _RATE_LIMIT_PATTERNS
assert "throttlingexception" in _RATE_LIMIT_PATTERNS
def test_context_overflow_patterns(self):
from agent.error_classifier import _CONTEXT_OVERFLOW_PATTERNS
assert "input is too long" in _CONTEXT_OVERFLOW_PATTERNS
# ---------------------------------------------------------------------------
# pyproject.toml bedrock extra
# ---------------------------------------------------------------------------
class TestPackaging:
"""Verify Bedrock remains a declared lazy optional dependency."""
@staticmethod
def _optional_dependencies():
import tomllib
from pathlib import Path
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
return tomllib.loads(content)["project"]["optional-dependencies"]
def test_bedrock_extra_exists(self):
extras = self._optional_dependencies()
assert "bedrock" in extras
assert any(dep.startswith("boto3==") for dep in extras["bedrock"])
def test_bedrock_is_not_eager_installed_by_all_extra(self):
extras = self._optional_dependencies()
assert "hermes-agent[bedrock]" not in extras["all"]
# ---------------------------------------------------------------------------
# Model ID dot preservation — regression for #11976
# ---------------------------------------------------------------------------
# AWS Bedrock inference-profile model IDs embed structural dots:
#
# global.anthropic.claude-opus-4-7
# us.anthropic.claude-sonnet-4-5-20250929-v1:0
# apac.anthropic.claude-haiku-4-5
#
# ``agent.anthropic_adapter.normalize_model_name`` converts dots to hyphens
# unless the caller opts in via ``preserve_dots=True``. Before this fix,
# ``AIAgent._anthropic_preserve_dots`` returned False for the ``bedrock``
# provider, so Claude-on-Bedrock requests went out with
# ``global-anthropic-claude-opus-4-7`` (all dots mangled to hyphens) and
# Bedrock rejected them with:
#
# HTTP 400: The provided model identifier is invalid.
#
# The fix adds ``bedrock`` to the preserve-dots provider allowlist and
# ``bedrock-runtime.`` to the base-URL heuristic, mirroring the shape of
# the opencode-go fix for #5211 (commit f77be22c), which extended this
# same allowlist.
class TestBedrockPreserveDotsFlag:
"""``AIAgent._anthropic_preserve_dots`` must return True on Bedrock so
inference-profile IDs survive the normalize step intact."""
def test_bedrock_provider_preserves_dots(self):
from types import SimpleNamespace
agent = SimpleNamespace(provider="bedrock", base_url="")
from run_agent import AIAgent
assert AIAgent._anthropic_preserve_dots(agent) is True
def test_bedrock_runtime_us_east_1_url_preserves_dots(self):
"""Defense-in-depth: even without an explicit ``provider="bedrock"``,
a ``bedrock-runtime.us-east-1.amazonaws.com`` base URL must not
mangle dots."""
from types import SimpleNamespace
agent = SimpleNamespace(
provider="custom",
base_url="https://bedrock-runtime.us-east-1.amazonaws.com",
)
from run_agent import AIAgent
assert AIAgent._anthropic_preserve_dots(agent) is True
def test_bedrock_runtime_ap_northeast_2_url_preserves_dots(self):
"""Reporter-reported region (ap-northeast-2) exercises the same
base-URL heuristic."""
from types import SimpleNamespace
agent = SimpleNamespace(
provider="custom",
base_url="https://bedrock-runtime.ap-northeast-2.amazonaws.com",
)
from run_agent import AIAgent
assert AIAgent._anthropic_preserve_dots(agent) is True
def test_non_bedrock_aws_url_does_not_preserve_dots(self):
"""Unrelated AWS endpoints (e.g. ``s3.us-east-1.amazonaws.com``)
must not accidentally activate the dot-preservation heuristic —
the heuristic is scoped to the ``bedrock-runtime.`` substring
specifically."""
from types import SimpleNamespace
agent = SimpleNamespace(
provider="custom",
base_url="https://s3.us-east-1.amazonaws.com",
)
from run_agent import AIAgent
assert AIAgent._anthropic_preserve_dots(agent) is False
def test_anthropic_native_still_does_not_preserve_dots(self):
"""Canary: adding Bedrock to the allowlist must not weaken the
existing Anthropic native behaviour — ``claude-sonnet-4.6`` still
becomes ``claude-sonnet-4-6`` for the Anthropic API."""
from types import SimpleNamespace
agent = SimpleNamespace(provider="anthropic", base_url="https://api.anthropic.com")
from run_agent import AIAgent
assert AIAgent._anthropic_preserve_dots(agent) is False
class TestBedrockModelNameNormalization:
"""End-to-end: ``normalize_model_name`` + the preserve-dots flag
reproduce the exact production request shape for each Bedrock model
family, confirming the fix resolves the reporter's HTTP 400."""
def test_global_anthropic_inference_profile_preserved(self):
"""The reporter's exact model ID."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name(
"global.anthropic.claude-opus-4-7", preserve_dots=True
) == "global.anthropic.claude-opus-4-7"
def test_us_anthropic_dated_inference_profile_preserved(self):
"""Regional + dated Sonnet inference profile."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name(
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
preserve_dots=True,
) == "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
def test_apac_anthropic_haiku_inference_profile_preserved(self):
"""APAC inference profile — same structural-dot shape."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name(
"apac.anthropic.claude-haiku-4-5", preserve_dots=True
) == "apac.anthropic.claude-haiku-4-5"
def test_bedrock_prefix_preserved_without_preserve_dots(self):
"""Bedrock inference profile IDs are auto-detected by prefix and
always returned unmangled -- ``preserve_dots`` is irrelevant for
these IDs because the dots are namespace separators, not version
separators. Regression for #12295."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name(
"global.anthropic.claude-opus-4-7", preserve_dots=False
) == "global.anthropic.claude-opus-4-7"
def test_bare_foundation_model_id_preserved(self):
"""Non-inference-profile Bedrock IDs
(e.g. ``anthropic.claude-3-5-sonnet-20241022-v2:0``) use dots as
vendor separators and must also survive intact under
``preserve_dots=True``."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name(
"anthropic.claude-3-5-sonnet-20241022-v2:0",
preserve_dots=True,
) == "anthropic.claude-3-5-sonnet-20241022-v2:0"
class TestBedrockBuildAnthropicKwargsEndToEnd:
"""Integration: calling ``build_anthropic_kwargs`` with a Bedrock-
shaped model ID and ``preserve_dots=True`` produces the unmangled
model string in the outgoing kwargs — the exact body sent to the
``bedrock-runtime.`` endpoint. This is the integration-level
regression for the reporter's HTTP 400."""
def test_bedrock_inference_profile_survives_build_kwargs(self):
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="global.anthropic.claude-opus-4-7",
messages=[{"role": "user", "content": "hi"}],
tools=None,
max_tokens=1024,
reasoning_config=None,
preserve_dots=True,
)
assert kwargs["model"] == "global.anthropic.claude-opus-4-7", (
"Bedrock inference-profile ID was mangled in build_anthropic_kwargs: "
f"{kwargs['model']!r}"
)
def test_bedrock_model_preserved_without_preserve_dots(self):
"""Bedrock inference profile IDs survive ``build_anthropic_kwargs``
even without ``preserve_dots=True`` -- the prefix auto-detection
in ``normalize_model_name`` is the load-bearing piece.
Regression for #12295."""
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="global.anthropic.claude-opus-4-7",
messages=[{"role": "user", "content": "hi"}],
tools=None,
max_tokens=1024,
reasoning_config=None,
preserve_dots=False,
)
assert kwargs["model"] == "global.anthropic.claude-opus-4-7"
class TestBedrockModelIdDetection:
"""Tests for ``_is_bedrock_model_id`` and the auto-detection that
makes ``normalize_model_name`` preserve dots for Bedrock IDs
regardless of ``preserve_dots``. Regression for #12295."""
def test_bare_bedrock_id_detected(self):
from agent.anthropic_adapter import _is_bedrock_model_id
assert _is_bedrock_model_id("anthropic.claude-opus-4-7") is True
def test_regional_us_prefix_detected(self):
from agent.anthropic_adapter import _is_bedrock_model_id
assert _is_bedrock_model_id("us.anthropic.claude-sonnet-4-5-v1:0") is True
def test_regional_global_prefix_detected(self):
from agent.anthropic_adapter import _is_bedrock_model_id
assert _is_bedrock_model_id("global.anthropic.claude-opus-4-7") is True
def test_regional_eu_prefix_detected(self):
from agent.anthropic_adapter import _is_bedrock_model_id
assert _is_bedrock_model_id("eu.anthropic.claude-sonnet-4-6") is True
def test_openrouter_format_not_detected(self):
from agent.anthropic_adapter import _is_bedrock_model_id
assert _is_bedrock_model_id("claude-opus-4.6") is False
def test_bare_claude_not_detected(self):
from agent.anthropic_adapter import _is_bedrock_model_id
assert _is_bedrock_model_id("claude-opus-4-7") is False
def test_bare_bedrock_id_preserved_without_flag(self):
"""The primary bug from #12295: ``anthropic.claude-opus-4-7``
sent to bedrock-mantle via auxiliary clients that don't pass
``preserve_dots=True``."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name(
"anthropic.claude-opus-4-7", preserve_dots=False
) == "anthropic.claude-opus-4-7"
def test_openrouter_dots_still_converted(self):
"""Non-Bedrock dotted model names must still be converted."""
from agent.anthropic_adapter import normalize_model_name
assert normalize_model_name("claude-opus-4.6") == "claude-opus-4-6"
def test_bare_bedrock_id_survives_build_kwargs(self):
"""End-to-end: bare Bedrock ID through ``build_anthropic_kwargs``
without ``preserve_dots=True`` -- the auxiliary client path."""
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="anthropic.claude-opus-4-7",
messages=[{"role": "user", "content": "hi"}],
tools=None,
max_tokens=1024,
reasoning_config=None,
preserve_dots=False,
)
assert kwargs["model"] == "anthropic.claude-opus-4-7"
# ---------------------------------------------------------------------------
# auxiliary_client Bedrock resolution — fix for #13919
# ---------------------------------------------------------------------------
# Before the fix, resolve_provider_client("bedrock", ...) fell through to the
# "unhandled auth_type" warning and returned (None, None), breaking all
# auxiliary tasks (compression, memory, summarization) for Bedrock users.
class TestAuxiliaryClientBedrockResolution:
"""Verify resolve_provider_client handles Bedrock's aws_sdk auth type."""
def test_bedrock_returns_client_with_credentials(self, monkeypatch):
"""With valid AWS credentials, Bedrock should return a usable client."""
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
monkeypatch.setenv("AWS_REGION", "us-west-2")
mock_anthropic_bedrock = MagicMock()
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
return_value=mock_anthropic_bedrock):
from agent.auxiliary_client import resolve_provider_client, AnthropicAuxiliaryClient
client, model = resolve_provider_client("bedrock", None)
assert client is not None, (
"resolve_provider_client('bedrock') returned None — "
"aws_sdk auth type is not handled"
)
assert isinstance(client, AnthropicAuxiliaryClient)
assert model is not None
assert client.api_key == "aws-sdk"
assert "us-west-2" in client.base_url
def test_bedrock_returns_none_without_credentials(self, monkeypatch):
"""Without AWS credentials, Bedrock should return (None, None) gracefully."""
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=False):
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("bedrock", None)
assert client is None
assert model is None
def test_bedrock_uses_configured_region(self, monkeypatch):
"""Bedrock client base_url should reflect AWS_REGION."""
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
monkeypatch.setenv("AWS_REGION", "eu-central-1")
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
return_value=MagicMock()):
from agent.auxiliary_client import resolve_provider_client
client, _ = resolve_provider_client("bedrock", None)
assert client is not None
assert "eu-central-1" in client.base_url
def test_bedrock_respects_explicit_model(self, monkeypatch):
"""When caller passes an explicit model, it should be used."""
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
return_value=MagicMock()):
from agent.auxiliary_client import resolve_provider_client
_, model = resolve_provider_client(
"bedrock", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
)
assert "claude-sonnet" in model
def test_bedrock_async_mode(self, monkeypatch):
"""Async mode should return an AsyncAnthropicAuxiliaryClient."""
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
return_value=MagicMock()):
from agent.auxiliary_client import resolve_provider_client, AsyncAnthropicAuxiliaryClient
client, model = resolve_provider_client("bedrock", None, async_mode=True)
assert client is not None
assert isinstance(client, AsyncAnthropicAuxiliaryClient)
def test_bedrock_default_model_is_haiku(self, monkeypatch):
"""Default auxiliary model for Bedrock should be Haiku (fast, cheap)."""
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
return_value=MagicMock()):
from agent.auxiliary_client import resolve_provider_client
_, model = resolve_provider_client("bedrock", None)
assert "haiku" in model.lower()
@@ -0,0 +1,253 @@
"""Regression guard: Codex Cloudflare 403 mitigation headers.
The ``chatgpt.com/backend-api/codex`` endpoint sits behind a Cloudflare layer
that whitelists a small set of first-party originators (``codex_cli_rs``,
``codex_vscode``, ``codex_sdk_ts``, ``Codex*``). Requests from non-residential
IPs (VPS, always-on servers, some corporate egress) that don't advertise an
allowed originator are served 403 with ``cf-mitigated: challenge`` regardless
of auth correctness.
``_codex_cloudflare_headers`` in ``agent.auxiliary_client`` centralizes the
header set so the primary chat client (``run_agent.AIAgent.__init__`` +
``_apply_client_headers_for_base_url``) and the auxiliary client paths
(``_build_codex_client`` and the ``raw_codex`` branch of ``resolve_provider_client``)
all emit the same headers.
These tests pin:
- the originator value (must be ``codex_cli_rs`` — the whitelisted one)
- the User-Agent shape (codex_cli_rs-prefixed)
- ``ChatGPT-Account-ID`` extraction from the OAuth JWT (canonical casing,
from codex-rs ``auth.rs``)
- graceful handling of malformed tokens (drop the account-ID header, don't
raise)
- primary-client wiring at both entry points in ``run_agent.py``
- aux-client wiring at both entry points in ``agent/auxiliary_client.py``
"""
from __future__ import annotations
import base64
import json
from unittest.mock import MagicMock, patch
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _make_codex_jwt(account_id: str = "acct-test-123") -> str:
"""Build a syntactically valid Codex-style JWT with the account_id claim."""
def b64url(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
header = b64url(b'{"alg":"RS256","typ":"JWT"}')
claims = {
"sub": "user-xyz",
"exp": 9999999999,
"https://api.openai.com/auth": {
"chatgpt_account_id": account_id,
"chatgpt_plan_type": "plus",
},
}
payload = b64url(json.dumps(claims).encode())
sig = b64url(b"fake-sig")
return f"{header}.{payload}.{sig}"
# ---------------------------------------------------------------------------
# _codex_cloudflare_headers — the shared helper
# ---------------------------------------------------------------------------
class TestCodexCloudflareHeaders:
def test_originator_is_codex_cli_rs(self):
"""Cloudflare whitelists codex_cli_rs — any other value is 403'd."""
from agent.auxiliary_client import _codex_cloudflare_headers
headers = _codex_cloudflare_headers(_make_codex_jwt())
assert headers["originator"] == "codex_cli_rs"
def test_user_agent_advertises_codex_cli_rs(self):
from agent.auxiliary_client import _codex_cloudflare_headers
headers = _codex_cloudflare_headers(_make_codex_jwt())
assert headers["User-Agent"].startswith("codex_cli_rs/")
def test_account_id_extracted_from_jwt(self):
from agent.auxiliary_client import _codex_cloudflare_headers
headers = _codex_cloudflare_headers(_make_codex_jwt("acct-abc-999"))
# Canonical casing — matches codex-rs auth.rs
assert headers["ChatGPT-Account-ID"] == "acct-abc-999"
def test_canonical_header_casing(self):
"""Upstream codex-rs uses PascalCase with trailing -ID. Match exactly."""
from agent.auxiliary_client import _codex_cloudflare_headers
headers = _codex_cloudflare_headers(_make_codex_jwt())
assert "ChatGPT-Account-ID" in headers
# The lowercase/titlecase variants MUST NOT be used — pin to be explicit
assert "chatgpt-account-id" not in headers
assert "ChatGPT-Account-Id" not in headers
def test_malformed_token_drops_account_id_without_raising(self):
from agent.auxiliary_client import _codex_cloudflare_headers
for bad in ["not-a-jwt", "", "only.one", " ", "...."]:
headers = _codex_cloudflare_headers(bad)
# Still returns base headers — never raises
assert headers["originator"] == "codex_cli_rs"
assert "ChatGPT-Account-ID" not in headers
def test_non_string_token_handled(self):
from agent.auxiliary_client import _codex_cloudflare_headers
headers = _codex_cloudflare_headers(None) # type: ignore[arg-type]
assert headers["originator"] == "codex_cli_rs"
assert "ChatGPT-Account-ID" not in headers
def test_jwt_without_chatgpt_account_id_claim(self):
"""A valid JWT that lacks the account_id claim should still return headers."""
from agent.auxiliary_client import _codex_cloudflare_headers
import base64 as _b64, json as _json
def b64url(data: bytes) -> str:
return _b64.urlsafe_b64encode(data).rstrip(b"=").decode()
payload = b64url(_json.dumps({"sub": "user-xyz", "exp": 9999999999}).encode())
token = f"{b64url(b'{}')}.{payload}.{b64url(b'sig')}"
headers = _codex_cloudflare_headers(token)
assert headers["originator"] == "codex_cli_rs"
assert "ChatGPT-Account-ID" not in headers
# ---------------------------------------------------------------------------
# Primary chat client wiring (run_agent.AIAgent)
# ---------------------------------------------------------------------------
class TestPrimaryClientWiring:
def test_init_wires_codex_headers_for_chatgpt_base_url(self):
from run_agent import AIAgent
token = _make_codex_jwt("acct-primary-init")
with patch("run_agent.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
AIAgent(
api_key=token,
base_url="https://chatgpt.com/backend-api/codex",
provider="openai-codex",
model="gpt-5.4",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
assert headers.get("originator") == "codex_cli_rs"
assert headers.get("ChatGPT-Account-ID") == "acct-primary-init"
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
def test_apply_client_headers_on_base_url_change(self):
"""Credential-rotation / base-url change path must also emit codex headers."""
from run_agent import AIAgent
token = _make_codex_jwt("acct-rotation")
with patch("run_agent.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
agent = AIAgent(
api_key="placeholder-openrouter-key",
base_url="https://openrouter.ai/api/v1",
provider="openrouter",
model="anthropic/claude-sonnet-4.6",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
# Simulate rotation into a Codex credential
agent._client_kwargs["api_key"] = token
agent._apply_client_headers_for_base_url(
"https://chatgpt.com/backend-api/codex"
)
headers = agent._client_kwargs.get("default_headers") or {}
assert headers.get("originator") == "codex_cli_rs"
assert headers.get("ChatGPT-Account-ID") == "acct-rotation"
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
def test_apply_client_headers_clears_codex_headers_off_chatgpt(self):
"""Switching AWAY from chatgpt.com must drop the codex headers."""
from run_agent import AIAgent
token = _make_codex_jwt()
with patch("run_agent.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
agent = AIAgent(
api_key=token,
base_url="https://chatgpt.com/backend-api/codex",
provider="openai-codex",
model="gpt-5.4",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
# Sanity: headers are set initially
assert "originator" in (agent._client_kwargs.get("default_headers") or {})
agent._apply_client_headers_for_base_url(
"https://api.anthropic.com"
)
# default_headers should be popped for anthropic base
assert "default_headers" not in agent._client_kwargs
def test_openrouter_base_url_does_not_get_codex_headers(self):
from run_agent import AIAgent
with patch("run_agent.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
AIAgent(
api_key="sk-or-test",
base_url="https://openrouter.ai/api/v1",
provider="openrouter",
model="anthropic/claude-sonnet-4.6",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
assert headers.get("originator") != "codex_cli_rs"
# ---------------------------------------------------------------------------
# Auxiliary client wiring (agent.auxiliary_client)
# ---------------------------------------------------------------------------
class TestAuxiliaryClientWiring:
def test_build_codex_client_passes_codex_headers(self, monkeypatch):
"""_build_codex_client builds the OpenAI client used for compression /
vision / title generation when routed through Codex. Must emit codex
headers."""
from agent import auxiliary_client
token = _make_codex_jwt("acct-aux-try-codex")
# Force _select_pool_entry to return "no pool" so we fall through to
# _read_codex_access_token.
monkeypatch.setattr(
auxiliary_client, "_select_pool_entry",
lambda provider: (False, None),
)
monkeypatch.setattr(
auxiliary_client, "_read_codex_access_token",
lambda: token,
)
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
client, model = auxiliary_client._build_codex_client("gpt-5.4")
assert client is not None
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
assert headers.get("originator") == "codex_cli_rs"
assert headers.get("ChatGPT-Account-ID") == "acct-aux-try-codex"
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
def test_resolve_provider_client_raw_codex_passes_codex_headers(self, monkeypatch):
"""The ``raw_codex=True`` branch (used by the main agent loop for direct
responses.stream() access) must also emit codex headers."""
from agent import auxiliary_client
token = _make_codex_jwt("acct-aux-raw-codex")
monkeypatch.setattr(
auxiliary_client, "_read_codex_access_token",
lambda: token,
)
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
client, model = auxiliary_client.resolve_provider_client(
"openai-codex", model="gpt-5.4", raw_codex=True,
)
assert client is not None
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
assert headers.get("originator") == "codex_cli_rs"
assert headers.get("ChatGPT-Account-ID") == "acct-aux-raw-codex"
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
+176
View File
@@ -0,0 +1,176 @@
from types import SimpleNamespace
import pytest
from agent.codex_responses_adapter import (
_format_responses_error,
_normalize_codex_response,
)
def test_normalize_codex_response_drops_transient_rs_tmp_reasoning_items():
response = SimpleNamespace(
status="completed",
output=[
SimpleNamespace(
type="reasoning",
id="rs_tmp_123",
encrypted_content="opaque-transient",
summary=[],
),
SimpleNamespace(
type="reasoning",
id="rs_456",
encrypted_content="opaque-stable",
summary=[SimpleNamespace(text="stable summary")],
),
SimpleNamespace(
type="message",
role="assistant",
status="completed",
content=[SimpleNamespace(type="output_text", text="done")],
),
],
)
assistant_message, finish_reason = _normalize_codex_response(response)
assert finish_reason == "stop"
assert assistant_message.content == "done"
assert assistant_message.codex_reasoning_items == [
{
"type": "reasoning",
"encrypted_content": "opaque-stable",
"id": "rs_456",
"summary": [{"type": "summary_text", "text": "stable summary"}],
}
]
def test_normalize_codex_response_treats_summary_only_reasoning_as_incomplete():
response = SimpleNamespace(
status="completed",
output=[
SimpleNamespace(
type="reasoning",
id="rs_tmp_789",
encrypted_content="opaque-transient",
summary=[SimpleNamespace(text="still thinking")],
)
],
)
assistant_message, finish_reason = _normalize_codex_response(response)
assert finish_reason == "incomplete"
assert assistant_message.content == ""
assert assistant_message.reasoning == "still thinking"
assert assistant_message.codex_reasoning_items is None
# ---------------------------------------------------------------------------
# _format_responses_error — adapted from anomalyco/opencode#28757.
# Provider failures should surface BOTH the code (rate_limit_exceeded /
# context_length_exceeded / internal_error / server_error) and the message,
# so consumers can tell rate limits apart from context-length failures and
# both apart from generic stream drops.
# ---------------------------------------------------------------------------
def test_format_responses_error_combines_code_and_message():
err = {"code": "rate_limit_exceeded", "message": "Slow down"}
assert _format_responses_error(err, "failed") == "rate_limit_exceeded: Slow down"
def test_format_responses_error_message_only():
err = {"message": "Upstream model unavailable"}
assert _format_responses_error(err, "failed") == "Upstream model unavailable"
def test_format_responses_error_code_only_when_message_empty():
# Some providers/proxies emit a code with an empty message body. We
# used to fall back to ``str(error_obj)`` — a dict dump — which leaked
# ``{'code': 'internal_error', 'message': ''}`` into chat output. Now
# the bare code is surfaced, which is the meaningful field.
err = {"code": "internal_error", "message": ""}
assert _format_responses_error(err, "failed") == "internal_error"
def test_format_responses_error_code_only_when_message_missing():
err = {"code": "server_error"}
assert _format_responses_error(err, "failed") == "server_error"
def test_format_responses_error_attribute_style_payload():
# SDK objects expose ``code``/``message`` as attributes rather than dict
# keys. The helper must accept both shapes since the Responses SDK
# returns SimpleNamespace-style objects on ``response.failed``.
err = SimpleNamespace(code="context_length_exceeded", message="too long")
assert _format_responses_error(err, "failed") == "context_length_exceeded: too long"
def test_format_responses_error_falls_back_to_status_when_empty():
assert (
_format_responses_error(None, "failed")
== "Responses API returned status 'failed'"
)
assert (
_format_responses_error(None, "cancelled")
== "Responses API returned status 'cancelled'"
)
def test_format_responses_error_stringifies_opaque_payload():
# Last-resort: a provider sent something that isn't a dict and has no
# code/message attributes. Surface its repr rather than swallow it
# silently — at least it's visible in logs.
assert _format_responses_error("opaque sentinel", "failed") == "opaque sentinel"
def test_format_responses_error_ignores_non_string_code_message():
# Defensive: a malformed gateway could send numbers/objects in these
# fields. We don't want to crash; we want a best-effort string.
err = {"code": 500, "message": None}
assert _format_responses_error(err, "failed") == "500"
def test_normalize_codex_response_failed_includes_code_in_error():
"""Regression: response_status == 'failed' should surface the error
code, not just the message. Used to leak a bare 'Slow down' string
that was indistinguishable from a generic stream truncation."""
# ``output`` non-empty so we don't trip the "no output items" guard
# before reaching the failed-status branch. Real failed responses
# often DO carry a partial message item alongside the error.
response = SimpleNamespace(
status="failed",
output=[
SimpleNamespace(
type="message",
role="assistant",
status="incomplete",
content=[SimpleNamespace(type="output_text", text="partial")],
),
],
error={"code": "rate_limit_exceeded", "message": "Slow down"},
)
with pytest.raises(RuntimeError, match=r"^rate_limit_exceeded: Slow down$"):
_normalize_codex_response(response)
def test_normalize_codex_response_failed_with_message_only():
"""Backwards-compat: a failed response with only a message field
(no code) should still surface that message verbatim."""
response = SimpleNamespace(
status="failed",
output=[
SimpleNamespace(
type="message",
role="assistant",
status="incomplete",
content=[SimpleNamespace(type="output_text", text="partial")],
),
],
error={"message": "model error"},
)
with pytest.raises(RuntimeError, match=r"^model error$"):
_normalize_codex_response(response)
+425
View File
@@ -0,0 +1,425 @@
"""Regression tests for the Codex time-to-first-byte (TTFB) watchdog.
The chatgpt.com/backend-api/codex endpoint has an intermittent failure mode
where it accepts the connection but never emits a single stream event. The
watchdog in ``interruptible_api_call`` kills such a connection at a short TTFB
cutoff (instead of waiting out the much longer wall-clock stale timeout) so the
retry loop can reconnect promptly. Once any stream event arrives, the TTFB
watchdog is satisfied and a separate idle watchdog handles streams that stop
emitting SSE events.
The "bytes flowing" signal is ``agent._codex_stream_last_event_ts``, set on
*any* event by ``codex_runtime.run_codex_stream`` — so reasoning-only or
tool-call-only turns (which emit no output-text deltas) are not mistaken for a
stall.
"""
from __future__ import annotations
import sys
import time
import types
from types import SimpleNamespace
import pytest
# Stub optional heavy imports so run_agent imports cleanly in isolation.
sys.modules.setdefault("fire", types.SimpleNamespace(Fire=lambda *a, **k: None))
sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
sys.modules.setdefault("fal_client", types.SimpleNamespace())
def _make_codex_agent(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
(tmp_path / "config.yaml").write_text("{}\n", encoding="utf-8")
from run_agent import AIAgent
agent = AIAgent(
model="gpt-5.5",
provider="openai-codex",
api_key="sk-dummy",
base_url="https://chatgpt.com/backend-api/codex",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
platform="cli",
)
# The watchdog is gated on the codex_responses api_mode; assert/force it so
# the test is robust to detection-logic changes elsewhere.
agent.api_mode = "codex_responses"
monkeypatch.setattr(agent, "_emit_status", lambda *a, **k: None)
# Keep the wall-clock stale timeout high so any early kill is unambiguously
# the TTFB path, not the stale-call path.
monkeypatch.setattr(
agent, "_compute_non_stream_stale_timeout", lambda *a, **k: 60.0
)
return agent
def test_ttfb_kills_when_no_stream_event(tmp_path, monkeypatch):
"""Backend accepts the connection but emits no event -> killed at the TTFB
cutoff, well before the 60s wall-clock stale timeout, with a retryable
TimeoutError and a ``codex_ttfb_kill`` close reason."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
# Never set _codex_stream_last_event_ts: simulate zero events arriving.
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
t0 = time.time()
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
elapsed = time.time() - t0
assert "TTFB" in str(excinfo.value)
assert "codex_ttfb_kill" in closes
# ~1s cutoff + 2s join grace; must be far under the 60s stale timeout.
assert elapsed < 15, f"TTFB watchdog took {elapsed:.1f}s"
finally:
stop["flag"] = True
def test_ttfb_default_tolerates_slow_first_event(tmp_path, monkeypatch):
"""With no env var set, the no-byte TTFB default is generous (120s), so a
request whose first stream event is merely slow (~2s of backend admission /
prefill) is NOT killed. This is the subscription-backed Codex case the tight
12s default used to abort mid-prefill."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
# Default behavior: no explicit TTFB override.
monkeypatch.delenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", raising=False)
monkeypatch.delenv("HERMES_CODEX_TTFB_MAX_SECONDS", raising=False)
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
sentinel = SimpleNamespace(ok=True)
def fake_slow_first_event(api_kwargs, client=None, on_first_delta=None):
# Backend is alive but slow to admit: first event lands after ~2s,
# well under the 120s default cutoff. Mark the first byte so the
# no-byte detector sees activity, then return the response.
time.sleep(2.0)
agent._codex_stream_last_event_ts = time.time()
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_slow_first_event)
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_ttfb_includes_silent_hang_hint_for_gpt_5_5(tmp_path, monkeypatch):
"""The no-first-byte watchdog should surface the same actionable hint as the
stale-call timeout path when the model matches the silent-hang heuristic."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
statuses: list[str] = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(agent, "_buffer_status", lambda msg: statuses.append(msg))
monkeypatch.setattr(agent, "_emit_status", lambda msg: statuses.append(msg))
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
message = str(excinfo.value)
assert "gpt-5.4" in message
assert "gpt-5.3-codex" in message
assert "gpt-5.4-codex" in message
assert "codex_ttfb_kill" in closes
assert statuses, "expected a user-facing watchdog status"
assert any("gpt-5.4" in s and "gpt-5.3-codex" in s for s in statuses)
finally:
stop["flag"] = True
def test_ttfb_high_env_is_capped_for_openai_codex(tmp_path, monkeypatch):
"""A stale local env value like 90s must not make openai-codex wait 90s
before reconnecting when the backend emits no SSE frames."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "90")
monkeypatch.setenv("HERMES_CODEX_TTFB_MAX_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
t0 = time.time()
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.4", "input": "hi"})
elapsed = time.time() - t0
assert "TTFB threshold: 1s" in str(excinfo.value)
assert "codex_ttfb_kill" in closes
assert elapsed < 15, f"TTFB watchdog ignored cap and took {elapsed:.1f}s"
finally:
stop["flag"] = True
def test_ttfb_does_not_kill_when_events_flow(tmp_path, monkeypatch):
"""Once a stream event has arrived, a generation that runs past the TTFB
cutoff is NOT killed by the watchdog — it completes normally."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
sentinel = SimpleNamespace(ok=True)
def fake_stream(api_kwargs, client=None, on_first_delta=None):
# Bytes flowing: mark stream activity right away, then keep generating
# past the 1s TTFB cutoff before returning a real response.
agent._codex_stream_last_event_ts = time.time()
if on_first_delta:
on_first_delta()
time.sleep(2.0)
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_event_idle_kills_after_first_event_then_silence(tmp_path, monkeypatch):
"""If Codex emits an opening SSE event and then goes silent, kill it via
the stream-idle watchdog instead of waiting for the long non-stream stale
timeout."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "10")
monkeypatch.setenv("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent,
"_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent,
"_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_stream(api_kwargs, client=None, on_first_delta=None):
agent._codex_stream_last_event_ts = time.time()
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert "after first byte" in str(excinfo.value)
assert "codex_stream_idle_kill" in closes
assert "codex_ttfb_kill" not in closes
finally:
stop["flag"] = True
def test_ttfb_disabled_via_env_zero(tmp_path, monkeypatch):
"""Setting HERMES_CODEX_TTFB_TIMEOUT_SECONDS=0 disables the TTFB watchdog;
a no-event stall then falls through to the (here, 60s) stale timeout, so a
short hang is NOT killed by TTFB."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "0")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
sentinel = SimpleNamespace(ok=True)
def fake_stream(api_kwargs, client=None, on_first_delta=None):
# No event marker, but only briefly — well under the 60s stale timeout.
time.sleep(2.0)
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_large_codex_request_waits_instead_of_ttfb_reconnect(tmp_path, monkeypatch):
"""Large Codex inputs can legitimately take longer than the small-request
first-byte cutoff before the first SSE frame. Preserve the full input and
wait instead of killing/retrying at TTFB."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client", lambda c, reason=None: closes.append(reason)
)
monkeypatch.setattr(
agent, "_close_request_openai_client", lambda c, reason=None: closes.append(reason)
)
sentinel = SimpleNamespace(ok=True)
def fake_stream(api_kwargs, client=None, on_first_delta=None):
# No event marker for 2s: this would trip the 1s TTFB watchdog on a
# small request, but should be allowed for a large request.
time.sleep(2.0)
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
large_input = "x" * 120_000 # ~30k estimated tokens, above large-request gate.
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": large_input})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_large_codex_request_strict_ttfb_env_still_reconnects(tmp_path, monkeypatch):
"""Operators can force the old early-reconnect behavior for large inputs
with HERMES_CODEX_TTFB_STRICT=1."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
monkeypatch.setenv("HERMES_CODEX_TTFB_STRICT", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client", lambda c, reason=None: closes.append(reason)
)
monkeypatch.setattr(
agent, "_close_request_openai_client", lambda c, reason=None: closes.append(reason)
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
large_input = "x" * 120_000
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": large_input})
assert "TTFB threshold: 1s" in str(excinfo.value)
assert "codex_ttfb_kill" in closes
finally:
stop["flag"] = True
+144
View File
@@ -0,0 +1,144 @@
"""Tests for focus_topic flowing through the compressor.
Verifies that _generate_summary and compress accept and use the focus_topic
parameter correctly. Inspired by Claude Code's /compact <focus>.
"""
from unittest.mock import MagicMock, patch
from agent.context_compressor import ContextCompressor
def _make_compressor():
"""Create a ContextCompressor with minimal state for testing."""
compressor = ContextCompressor.__new__(ContextCompressor)
compressor.protect_first_n = 2
compressor.protect_last_n = 5
compressor.tail_token_budget = 20000
compressor.context_length = 200000
compressor.threshold_percent = 0.80
compressor.threshold_tokens = 160000
compressor.max_summary_tokens = 10000
compressor.quiet_mode = True
compressor.compression_count = 0
compressor.last_prompt_tokens = 0
compressor._previous_summary = None
compressor._summary_failure_cooldown_until = 0.0
compressor.summary_model = None
compressor.model = "test-model"
compressor.provider = "test"
compressor.base_url = "http://localhost"
compressor.api_key = "test-key"
compressor.api_mode = "chat_completions"
return compressor
def test_focus_topic_injected_into_summary_prompt():
"""When focus_topic is provided, the LLM prompt includes focus guidance."""
compressor = _make_compressor()
turns = [
{"role": "user", "content": "Tell me about the database schema"},
{"role": "assistant", "content": "The schema has tables: users, orders, products."},
]
captured_prompt = {}
def mock_call_llm(**kwargs):
captured_prompt["messages"] = kwargs["messages"]
resp = MagicMock()
resp.choices = [MagicMock()]
resp.choices[0].message.content = "## Goal\nUnderstand DB schema."
return resp
with patch("agent.context_compressor.call_llm", mock_call_llm):
result = compressor._generate_summary(turns, focus_topic="database schema")
assert result is not None
prompt_text = captured_prompt["messages"][0]["content"]
assert 'FOCUS TOPIC: "database schema"' in prompt_text
assert "PRIORITISE" in prompt_text
assert "60-70%" in prompt_text
def test_no_focus_topic_no_injection():
"""Without focus_topic, the prompt doesn't contain focus guidance."""
compressor = _make_compressor()
turns = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi"},
]
captured_prompt = {}
def mock_call_llm(**kwargs):
captured_prompt["messages"] = kwargs["messages"]
resp = MagicMock()
resp.choices = [MagicMock()]
resp.choices[0].message.content = "## Goal\nGreeting."
return resp
with patch("agent.context_compressor.call_llm", mock_call_llm):
result = compressor._generate_summary(turns)
prompt_text = captured_prompt["messages"][0]["content"]
assert "FOCUS TOPIC" not in prompt_text
def test_compress_passes_focus_to_generate_summary():
"""compress() passes focus_topic through to _generate_summary."""
compressor = _make_compressor()
# Track what _generate_summary receives
received_kwargs = {}
original_generate = compressor._generate_summary
def tracking_generate(turns, **kwargs):
received_kwargs.update(kwargs)
return "## Goal\nTest."
compressor._generate_summary = tracking_generate
messages = [
{"role": "system", "content": "System prompt"},
{"role": "user", "content": "first"},
{"role": "assistant", "content": "reply1"},
{"role": "user", "content": "second"},
{"role": "assistant", "content": "reply2"},
{"role": "user", "content": "third"},
{"role": "assistant", "content": "reply3"},
{"role": "user", "content": "fourth"},
{"role": "assistant", "content": "reply4"},
]
compressor.compress(messages, current_tokens=100000, focus_topic="authentication flow")
assert received_kwargs.get("focus_topic") == "authentication flow"
def test_compress_none_focus_by_default():
"""compress() passes None focus_topic by default."""
compressor = _make_compressor()
received_kwargs = {}
def tracking_generate(turns, **kwargs):
received_kwargs.update(kwargs)
return "## Goal\nTest."
compressor._generate_summary = tracking_generate
messages = [
{"role": "system", "content": "System prompt"},
{"role": "user", "content": "first"},
{"role": "assistant", "content": "reply1"},
{"role": "user", "content": "second"},
{"role": "assistant", "content": "reply2"},
{"role": "user", "content": "third"},
{"role": "assistant", "content": "reply3"},
{"role": "user", "content": "fourth"},
{"role": "assistant", "content": "reply4"},
]
compressor.compress(messages, current_tokens=100000)
assert received_kwargs.get("focus_topic") is None
@@ -0,0 +1,240 @@
"""Regression: prevent transcript fork when two paths compress the same session_id.
Damien's incident (Discord, 2026-05-28): a long Hermes session in a Discord
gateway hit the compression threshold at the end of a turn. The parent agent
finished delivering the response and ``conversation_loop.py`` fired
``_spawn_background_review(...)`` — which builds a forked ``AIAgent`` that
inherits ``agent.session_id`` (see ``agent/background_review.py``::
``review_agent.session_id = agent.session_id``). Roughly two seconds later
a synthetic ``Background process proc_… completed`` event arrived and
started a fresh turn on the same parent ``session_id`` (still cached in the
gateway's ``SessionEntry``). Both paths hit preflight compression on the
same parent transcript and called ``_compress_context`` concurrently. Each
ended the parent and created its own CHILD session in ``state.db``, both
parented to the same old id. The gateway's ``SessionEntry`` only caught one
rotation; the other child became an orphan that silently accumulated writes.
Repro shape on Damien's machine:
parent 20260527_234659_e65f0e ended_at=set end_reason='compression'
child 20260528_113619_fc80e1 parent=20260527_234659_e65f0e (in SessionEntry)
child <orphan> parent=20260527_234659_e65f0e (silent writes)
This regression simulates the two concurrent ``compress_context`` calls
against a shared ``state.db`` and asserts that the per-session compression
lock added in this PR prevents the orphan child. Without the lock the
fixture deterministically produces 2 children; with the lock, exactly 1.
"""
from __future__ import annotations
import os
import threading
import time
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from hermes_state import SessionDB
def _build_agent_with_db(db: SessionDB, session_id: str):
"""Build an AIAgent that's wired to ``db`` and pinned to ``session_id``."""
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://openrouter.ai/api/v1",
model="test/model",
quiet_mode=True,
session_db=db,
session_id=session_id,
skip_context_files=True,
skip_memory=True,
)
# Stub the compressor so it returns deterministic output and DOESN'T make
# an LLM call. Sleep inside compress() so the two threads' rotations
# actually overlap — without that the OS could happen to serialize them
# and hide the bug.
compressor = MagicMock()
def _compress_with_overlap(*_a, **_kw):
time.sleep(0.25)
return [
{"role": "user", "content": "[CONTEXT COMPACTION] summary"},
{"role": "user", "content": "tail"},
]
compressor.compress.side_effect = _compress_with_overlap
compressor.compression_count = 1
compressor.last_prompt_tokens = 0
compressor.last_completion_tokens = 0
compressor._last_summary_error = None
compressor._last_compress_aborted = False
compressor._last_aux_model_failure_model = None
compressor._last_aux_model_failure_error = None
agent.context_compressor = compressor
return agent
def _count_children(db: SessionDB, parent_sid: str) -> int:
"""Count rows in state.db whose parent_session_id == parent_sid."""
rows = db._conn.execute(
"SELECT id FROM sessions WHERE parent_session_id = ?",
(parent_sid,),
).fetchall()
return len(rows)
def test_concurrent_compression_does_not_fork_session(tmp_path: Path) -> None:
"""Two AIAgents that share a session_id MUST NOT both rotate it.
Without the per-session compression lock this fixture deterministically
produces 2 child sessions (transcript fork). With the lock the second
path aborts cleanly, leaving exactly 1 canonical child.
"""
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "PARENT_TEST_SESSION"
db.create_session(parent_sid, source="discord")
# Two agents on the same session_id, both wired to the same db —
# mirrors the parent-turn agent + the background-review fork right
# after a turn ends.
agent_a = _build_agent_with_db(db, parent_sid)
agent_b = _build_agent_with_db(db, parent_sid)
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
def run(agent):
try:
agent._compress_context(messages, "sys", approx_tokens=120_000)
except Exception:
# Surface to the test if either raises — should not happen.
raise
t_a = threading.Thread(target=run, args=(agent_a,), name="main_turn")
t_b = threading.Thread(target=run, args=(agent_b,), name="review_fork")
t_a.start()
t_b.start()
t_a.join(timeout=10)
t_b.join(timeout=10)
# Exactly one canonical child — not two orphans.
assert _count_children(db, parent_sid) == 1, (
"Compression lock failed: parent session has multiple children in state.db "
"(transcript fork). This is Damien's incident shape — see the test docstring."
)
# And exactly one of the two agents actually rotated its session_id; the
# other should still hold the parent_sid (its compression was skipped).
rotated = sum(
1 for a in (agent_a, agent_b) if a.session_id != parent_sid
)
assert rotated == 1, (
f"Expected exactly one agent to rotate session_id, got {rotated}. "
"Both agents rotating means the lock didn't serialize them."
)
# The lock must be released after the winner finished.
assert db.get_compression_lock_holder(parent_sid) is None, (
"Compression lock leaked: still held after both rotations completed."
)
def test_skipped_compression_returns_messages_unchanged(tmp_path: Path) -> None:
"""The loser of the lock race must return its input messages verbatim.
Callers (preflight compression in ``conversation_loop.py``) detect the
no-op via ``len(returned) == len(input)`` and stop the auto-compress
retry loop. If the skipped path returned the compressed view, that
detection would break and the caller would mutate the conversation
without going through state.db rotation.
"""
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "LOSER_TEST"
db.create_session(parent_sid, source="discord")
# Pre-acquire the lock so the agent's compress_context sees it held.
held = db.try_acquire_compression_lock(parent_sid, "external_holder")
assert held is True
agent = _build_agent_with_db(db, parent_sid)
messages = [{"role": "user", "content": "m1"}, {"role": "user", "content": "m2"}]
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
# Skipped: messages returned verbatim, no rotation
assert compressed is messages or compressed == messages
assert agent.session_id == parent_sid
# Compressor was never called (the skip happens before .compress())
agent.context_compressor.compress.assert_not_called()
class _NoLockSubsystemDB:
"""Wraps a real SessionDB but simulates a pre-#34351 version skew.
A long-lived process can hold ``hermes_state.SessionDB`` bound to the
OLD class in memory (no compression-lock methods) while a lazily
re-imported ``conversation_compression.py`` calls the NEW lock code.
``try_acquire_compression_lock`` then raises ``AttributeError`` — which
is NOT a ``sqlite3.Error``, so the method's own fail-open guard never
runs. Before the fix the exception propagated to the outer agent loop,
which printed the error and retried; compression never succeeded, the
token count never dropped, and the loop re-triggered compaction forever.
"""
def __init__(self, real_db: SessionDB) -> None:
self._real = real_db
def try_acquire_compression_lock(self, *_a, **_k): # noqa: D401
raise AttributeError(
"'SessionDB' object has no attribute 'try_acquire_compression_lock'"
)
def get_compression_lock_holder(self, *_a, **_k):
raise AttributeError("'SessionDB' object has no attribute 'get_compression_lock_holder'")
def release_compression_lock(self, *_a, **_k):
raise AttributeError("'SessionDB' object has no attribute 'release_compression_lock'")
def __getattr__(self, name):
# Everything else (create_session, append, rotation helpers) goes to
# the real db so the post-lock compression + rotation path runs.
return getattr(self._real, name)
def test_missing_lock_subsystem_fails_open_not_infinite_loop(tmp_path: Path) -> None:
"""Version skew (no lock methods) must fail OPEN, not raise into the loop.
Reproduces the "API call #47/#48/#49 ... has no attribute
try_acquire_compression_lock" infinite-compaction spin: when the lock
subsystem is absent, ``_compress_context`` must skip locking and proceed
with compression (so the loop makes progress and terminates) instead of
letting the ``AttributeError`` escape to the retry loop.
"""
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "SKEW_TEST_SESSION"
db.create_session(parent_sid, source="discord")
agent = _build_agent_with_db(db, parent_sid)
# Swap in the lock-less wrapper AFTER construction (the agent already
# holds a normal db reference; we only break the lock methods).
agent._session_db = _NoLockSubsystemDB(db)
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
# MUST NOT raise AttributeError. Before the fix this raised and the
# outer loop would retry forever.
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
# Compression actually ran (proceeded past the broken lock) and made
# progress, so the auto-compress loop would terminate.
agent.context_compressor.compress.assert_called_once()
assert len(compressed) < len(messages), (
"Compression made no progress despite failing open — loop would still spin."
)
# Session rotated (compression succeeded end-to-end).
assert agent.session_id != parent_sid
@@ -0,0 +1,266 @@
"""Tests for post-compression historical-media stripping.
Port of Kilo-Org/kilocode#9434 (adapted for OpenAI-style message lists).
Without this pass, tail messages keep their original multi-MB base-64 image
payloads after context compression, and every subsequent request re-ships
them — sometimes breaching provider body-size limits and wedging the
session.
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
from agent.context_compressor import (
ContextCompressor,
_content_has_images,
_is_image_part,
_strip_historical_media,
_strip_images_from_content,
)
IMG_URL = {
"type": "image_url",
"image_url": {"url": "data:image/png;base64," + ("A" * 1024)},
}
INPUT_IMG = {
"type": "input_image",
"image_url": "data:image/png;base64," + ("B" * 1024),
}
ANTHROPIC_IMG = {
"type": "image",
"source": {"type": "base64", "media_type": "image/png", "data": "C" * 1024},
}
TEXT = {"type": "text", "text": "hi"}
INPUT_TEXT = {"type": "input_text", "text": "hi"}
class TestIsImagePart:
def test_openai_chat_shape(self):
assert _is_image_part(IMG_URL) is True
def test_openai_responses_shape(self):
assert _is_image_part(INPUT_IMG) is True
def test_anthropic_native_shape(self):
assert _is_image_part(ANTHROPIC_IMG) is True
def test_text_part_is_not_image(self):
assert _is_image_part(TEXT) is False
assert _is_image_part(INPUT_TEXT) is False
def test_non_dict_rejected(self):
assert _is_image_part("image") is False
assert _is_image_part(None) is False
assert _is_image_part(42) is False
class TestContentHasImages:
def test_string_content(self):
assert _content_has_images("a string") is False
def test_empty_list(self):
assert _content_has_images([]) is False
def test_text_only_list(self):
assert _content_has_images([TEXT, TEXT]) is False
def test_list_with_image(self):
assert _content_has_images([TEXT, IMG_URL]) is True
def test_none(self):
assert _content_has_images(None) is False
class TestStripImagesFromContent:
def test_string_passthrough(self):
assert _strip_images_from_content("hello") == "hello"
def test_none_passthrough(self):
assert _strip_images_from_content(None) is None
def test_text_only_passthrough(self):
parts = [TEXT, {"type": "text", "text": "world"}]
assert _strip_images_from_content(parts) == parts
def test_replaces_image_with_placeholder(self):
parts = [TEXT, IMG_URL]
out = _strip_images_from_content(parts)
assert len(out) == 2
assert out[0] == TEXT
assert out[1] == {
"type": "text",
"text": "[Attached image — stripped after compression]",
}
def test_does_not_mutate_input(self):
parts = [IMG_URL, TEXT]
_ = _strip_images_from_content(parts)
assert parts[0] is IMG_URL # original list untouched
assert parts[1] is TEXT
def test_handles_all_three_shapes(self):
parts = [IMG_URL, INPUT_IMG, ANTHROPIC_IMG, TEXT]
out = _strip_images_from_content(parts)
assert sum(1 for p in out if p.get("type") == "text") == 4
assert not any(_is_image_part(p) for p in out)
class TestStripHistoricalMedia:
def test_empty_passthrough(self):
assert _strip_historical_media([]) == []
def test_no_images_anywhere(self):
msgs = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "hey"},
{"role": "user", "content": "bye"},
]
assert _strip_historical_media(msgs) is msgs # identity — no copy
def test_single_image_user_only_first_message(self):
# Only image-bearing user is the first message — nothing before it.
msgs = [
{"role": "user", "content": [TEXT, IMG_URL]},
{"role": "assistant", "content": "ok"},
]
out = _strip_historical_media(msgs)
assert out is msgs # no-op
# Image still there.
assert _content_has_images(out[0]["content"])
def test_strips_older_user_image_keeps_newest(self):
msgs = [
{"role": "user", "content": [TEXT, IMG_URL]}, # old — strip
{"role": "assistant", "content": "looked at it"},
{"role": "user", "content": [TEXT, INPUT_IMG]}, # newest — keep
]
out = _strip_historical_media(msgs)
assert out is not msgs # new list
# First message's image was replaced
assert not _content_has_images(out[0]["content"])
# Newest user still has its image
assert _content_has_images(out[2]["content"])
def test_strips_assistant_and_tool_images_before_anchor(self):
msgs = [
{"role": "user", "content": [TEXT, IMG_URL]}, # old user
{"role": "assistant", "content": [TEXT, IMG_URL]}, # old assistant
{"role": "tool", "content": [TEXT, IMG_URL], "tool_call_id": "t1"},
{"role": "user", "content": [TEXT, IMG_URL]}, # newest user — keep
]
out = _strip_historical_media(msgs)
for i in range(3):
assert not _content_has_images(out[i]["content"]), f"msg {i} still has image"
assert _content_has_images(out[3]["content"])
def test_text_only_newest_user_still_strips_older_images(self):
# The anchor is "newest user WITH images". If the newest user is
# text-only, we fall back to the previous image-bearing user turn.
msgs = [
{"role": "user", "content": [TEXT, IMG_URL]},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": [TEXT, IMG_URL]}, # anchor
{"role": "assistant", "content": "done"},
{"role": "user", "content": "follow-up text only"},
]
out = _strip_historical_media(msgs)
# First image-bearing user (index 0) was stripped — it was before the
# newest image-bearing user (index 2).
assert not _content_has_images(out[0]["content"])
# Anchor (index 2) keeps its image.
assert _content_has_images(out[2]["content"])
def test_no_image_bearing_user_is_noop(self):
msgs = [
{"role": "user", "content": "first"},
{"role": "assistant", "content": [TEXT, IMG_URL]}, # assistant image only
{"role": "user", "content": "second"},
]
out = _strip_historical_media(msgs)
# No image-bearing user anchor → no stripping.
assert out is msgs
assert _content_has_images(out[1]["content"])
def test_does_not_mutate_input_messages(self):
msg0 = {"role": "user", "content": [TEXT, IMG_URL]}
msg1 = {"role": "user", "content": [TEXT, IMG_URL]}
msgs = [msg0, msg1]
_ = _strip_historical_media(msgs)
# Originals untouched
assert _content_has_images(msg0["content"])
assert _content_has_images(msg1["content"])
def test_idempotent(self):
msgs = [
{"role": "user", "content": [TEXT, IMG_URL]},
{"role": "assistant", "content": "k"},
{"role": "user", "content": [TEXT, IMG_URL]},
]
first = _strip_historical_media(msgs)
second = _strip_historical_media(first)
# Second pass is a no-op — no images left before the anchor.
assert second is first
def test_non_dict_messages_pass_through(self):
msgs = [
"not-a-dict", # shouldn't crash
{"role": "user", "content": [TEXT, IMG_URL]},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": [TEXT, IMG_URL]},
]
out = _strip_historical_media(msgs)
assert out[0] == "not-a-dict"
# Image-bearing user at index 1 is before the anchor (index 3) → stripped.
assert not _content_has_images(out[1]["content"])
class TestCompressIntegration:
"""Verify the stripping runs inside ContextCompressor.compress()."""
@pytest.fixture
def compressor(self):
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
c = ContextCompressor(
model="test/model",
threshold_percent=0.50,
protect_first_n=1,
protect_last_n=2,
quiet_mode=True,
)
return c
def test_compress_strips_historical_images(self, compressor):
# Enough messages to trigger the summarize path. protect_first_n=1 +
# protect_last_n=2 + a middle window of at least 3 with a summary.
msgs = [
{"role": "system", "content": "sys"},
{"role": "user", "content": [TEXT, IMG_URL]}, # old image-bearing user
{"role": "assistant", "content": "looked at it"},
{"role": "user", "content": "follow-up"},
{"role": "assistant", "content": "ack"},
{"role": "user", "content": "more"},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": [TEXT, IMG_URL]}, # newest image-bearing user (tail)
{"role": "assistant", "content": "done"},
]
# Bypass the real LLM summary — return a stub so compress() proceeds.
with patch.object(compressor, "_generate_summary", return_value="SUMMARY TEXT"):
out = compressor.compress(msgs, current_tokens=60_000)
# Newest user turn with image should still have it (it's in the tail).
user_imgs = [m for m in out if m.get("role") == "user" and _content_has_images(m.get("content"))]
assert len(user_imgs) == 1, (
"Expected exactly one user message with images after compression "
f"(the newest one); got {len(user_imgs)}"
)
# No assistant or tool messages should carry images either.
for m in out:
if m is user_imgs[0]:
continue
assert not _content_has_images(m.get("content")), (
f"Stale image in {m.get('role')!r} message after compression"
)
+140
View File
@@ -0,0 +1,140 @@
"""Tests for image-token accounting in the context compressor.
Covers the native-image-routing PR's companion change: the compressor's
multimodal message length counter now charges ~1600 tokens per attached
image part instead of 0, so tail-cut / prune decisions are accurate for
creative workflows that iterate on images across many turns.
"""
from __future__ import annotations
from agent.context_compressor import (
_CHARS_PER_TOKEN,
_IMAGE_CHAR_EQUIVALENT,
_IMAGE_TOKEN_ESTIMATE,
_content_length_for_budget,
)
class TestContentLengthForBudget:
def test_plain_string(self):
assert _content_length_for_budget("hello world") == 11
def test_empty_string(self):
assert _content_length_for_budget("") == 0
def test_none_coerces_to_zero(self):
assert _content_length_for_budget(None) == 0
def test_text_only_list(self):
content = [
{"type": "text", "text": "first"},
{"type": "text", "text": "second"},
]
assert _content_length_for_budget(content) == 5 + 6
def test_single_image_part_charges_fixed_budget(self):
content = [
{"type": "text", "text": "look"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,XXXX"}},
]
# 4 chars of text + 1 image at fixed char-equivalent
assert _content_length_for_budget(content) == 4 + _IMAGE_CHAR_EQUIVALENT
def test_image_url_raw_base64_is_not_counted_as_chars(self):
"""A 1MB base64 blob inside an image_url must NOT inflate token count.
The flat image estimate is what the provider actually bills; the raw
base64 is transport payload, not context tokens.
"""
huge_url = "data:image/png;base64," + ("A" * 1_000_000)
content = [
{"type": "image_url", "image_url": {"url": huge_url}},
]
# Exactly one image's worth, not 1M + something.
assert _content_length_for_budget(content) == _IMAGE_CHAR_EQUIVALENT
def test_multiple_image_parts(self):
content = [
{"type": "text", "text": "compare"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,BBB"}},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,CCC"}},
]
assert _content_length_for_budget(content) == 7 + 3 * _IMAGE_CHAR_EQUIVALENT
def test_openai_responses_input_image_shape(self):
"""Responses API uses type=input_image with top-level image_url string."""
content = [
{"type": "input_text", "text": "hey"},
{"type": "input_image", "image_url": "data:image/png;base64,XX"},
]
# input_text has .text "hey" (3 chars) + 1 image
assert _content_length_for_budget(content) == 3 + _IMAGE_CHAR_EQUIVALENT
def test_anthropic_native_image_shape(self):
"""Anthropic native shape: {type: image, source: {...}}."""
content = [
{"type": "text", "text": "hi"},
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "XX"}},
]
assert _content_length_for_budget(content) == 2 + _IMAGE_CHAR_EQUIVALENT
def test_bare_string_part_in_list(self):
"""Older code paths sometimes produce mixed list-of-strings content."""
content = ["hello", {"type": "text", "text": "world"}]
assert _content_length_for_budget(content) == 5 + 5
def test_image_estimate_constant_is_reasonable(self):
"""Sanity-check the estimate aligns with real provider billing.
Anthropic ≈ width*height/750 → ~1600 for 1000×1200.
OpenAI GPT-4o high-detail 2048×2048 ≈ 1445.
Gemini 258/tile × 6 tiles for a 2048×2048 ≈ 1548.
Anything in the 800-2000 range is defensible. Enforce bounds so an
accidental edit doesn't drop it to e.g. 16.
"""
assert 800 <= _IMAGE_TOKEN_ESTIMATE <= 2500
assert _IMAGE_CHAR_EQUIVALENT == _IMAGE_TOKEN_ESTIMATE * _CHARS_PER_TOKEN
class TestTokenBudgetWithImages:
"""Integration: the compressor's tail-cut decision now respects image cost."""
def test_image_heavy_turns_count_toward_budget(self):
"""A tail with 5 image-bearing turns should blow past a 5K token budget."""
from agent.context_compressor import ContextCompressor
# Minimal compressor fixture — just enough to call _find_tail_cut_by_tokens
cc = object.__new__(ContextCompressor)
cc.tail_token_budget = 5000
# Build 10 messages: 5 with images, 5 with short text. Without the
# image-tokens fix, the compressor would think all 10 fit in 5K and
# protect them all. With the fix, images alone cost 5 × 1600 = 8K,
# so the tail should be trimmed.
messages = [{"role": "system", "content": "sys"}]
for i in range(5):
messages.append({
"role": "user",
"content": [
{"type": "text", "text": f"turn {i}"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},
],
})
messages.append({
"role": "assistant",
"content": f"response {i}",
})
cut = cc._find_tail_cut_by_tokens(messages, head_end=0, token_budget=5000)
# Budget is 5K, soft ceiling 7.5K. 5 images alone = 8000 image-tokens.
# Walking backward, the compressor should stop before including all 5.
# Exact cut depends on text lengths and min_tail, but it MUST be > 1
# (at least some head-side messages should be compressible).
assert cut > 1, (
f"Expected image-heavy tail to be trimmed; compressor placed cut at "
f"{cut} out of {len(messages)} (image tokens were likely ignored)."
)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,87 @@
"""Regression tests for iterative context-summary continuity."""
from unittest.mock import MagicMock, patch
from agent.context_compressor import ContextCompressor, SUMMARY_PREFIX
def _compressor() -> ContextCompressor:
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
return ContextCompressor(
model="test/model",
threshold_percent=0.85,
protect_first_n=1,
protect_last_n=1,
quiet_mode=True,
)
def _response(content: str):
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = content
return mock_response
def _messages_with_handoff(summary_body: str):
return [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{summary_body}"},
{"role": "assistant", "content": "handoff acknowledged after resume"},
{"role": "user", "content": "new user turn after resume"},
{"role": "assistant", "content": "new assistant work after resume"},
{"role": "user", "content": "more new work after resume"},
{"role": "assistant", "content": "latest tail response"},
{"role": "user", "content": "final active request stays in protected tail"},
]
def test_existing_previous_summary_is_not_serialized_again_as_new_turn():
"""Same-process iterative compression should not feed the old handoff twice."""
compressor = _compressor()
old_summary = "OLD-SUMMARY-BODY unique continuity facts"
compressor._previous_summary = old_summary
with patch("agent.context_compressor.call_llm", return_value=_response("updated summary")) as mock_call:
compressor.compress(_messages_with_handoff(old_summary))
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
assert "PREVIOUS SUMMARY:" in prompt
assert "NEW TURNS TO INCORPORATE:" in prompt
assert prompt.count(old_summary) == 1
assert f"[USER]: {SUMMARY_PREFIX}" not in prompt
def test_resume_rehydrates_previous_summary_from_handoff_message():
"""After restart/resume, the persisted handoff should regain summary identity."""
compressor = _compressor()
old_summary = "RESUMED-SUMMARY-BODY durable continuity facts"
assert compressor._previous_summary is None
with patch("agent.context_compressor.call_llm", return_value=_response("updated summary")) as mock_call:
compressor.compress(_messages_with_handoff(old_summary))
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
assert "PREVIOUS SUMMARY:" in prompt
assert "NEW TURNS TO INCORPORATE:" in prompt
assert "TURNS TO SUMMARIZE:" not in prompt
assert prompt.count(old_summary) == 1
assert f"[USER]: {SUMMARY_PREFIX}" not in prompt
def test_handoff_in_protected_head_populates_previous_summary_before_update():
"""A resumed protected-head handoff should restore iterative-summary state."""
compressor = _compressor()
old_summary = "PROTECTED-HEAD-SUMMARY durable facts from before restart"
seen_turns = []
def fake_generate_summary(turns_to_summarize, focus_topic=None):
seen_turns.extend(turns_to_summarize)
return "new summary from resumed turns"
with patch.object(compressor, "_generate_summary", side_effect=fake_generate_summary):
compressor.compress(_messages_with_handoff(old_summary))
assert compressor._previous_summary == old_summary
assert seen_turns
assert all(old_summary not in str(msg.get("content", "")) for msg in seen_turns)
+250
View File
@@ -0,0 +1,250 @@
"""Tests for the ContextEngine ABC and plugin slot."""
import json
import pytest
from typing import Any, Dict, List
from agent.context_engine import ContextEngine
from agent.context_compressor import ContextCompressor
# ---------------------------------------------------------------------------
# A minimal concrete engine for testing the ABC
# ---------------------------------------------------------------------------
class StubEngine(ContextEngine):
"""Minimal engine that satisfies the ABC without doing real work."""
def __init__(self, context_length=200000, threshold_pct=0.50):
self.context_length = context_length
self.threshold_tokens = int(context_length * threshold_pct)
self._compress_called = False
self._tools_called = []
@property
def name(self) -> str:
return "stub"
def update_from_response(self, usage: Dict[str, Any]) -> None:
self.last_prompt_tokens = usage.get("prompt_tokens", 0)
self.last_completion_tokens = usage.get("completion_tokens", 0)
self.last_total_tokens = usage.get("total_tokens", 0)
def should_compress(self, prompt_tokens: int = None) -> bool:
tokens = prompt_tokens if prompt_tokens is not None else self.last_prompt_tokens
return tokens >= self.threshold_tokens
def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) -> List[Dict[str, Any]]:
self._compress_called = True
self.compression_count += 1
# Trivial: just return as-is
return messages
def get_tool_schemas(self) -> List[Dict[str, Any]]:
return [
{
"name": "stub_search",
"description": "Search the stub engine",
"parameters": {"type": "object", "properties": {}},
}
]
def handle_tool_call(self, name: str, args: Dict[str, Any]) -> str:
self._tools_called.append(name)
return json.dumps({"ok": True, "tool": name})
# ---------------------------------------------------------------------------
# ABC contract tests
# ---------------------------------------------------------------------------
class TestContextEngineABC:
"""Verify the ABC enforces the required interface."""
def test_cannot_instantiate_abc_directly(self):
with pytest.raises(TypeError):
ContextEngine()
def test_missing_methods_raises(self):
"""A subclass missing required methods cannot be instantiated."""
class Incomplete(ContextEngine):
@property
def name(self):
return "incomplete"
with pytest.raises(TypeError):
Incomplete()
def test_stub_engine_satisfies_abc(self):
engine = StubEngine()
assert isinstance(engine, ContextEngine)
assert engine.name == "stub"
def test_compressor_is_context_engine(self):
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
assert isinstance(c, ContextEngine)
assert c.name == "compressor"
# ---------------------------------------------------------------------------
# Default method behavior
# ---------------------------------------------------------------------------
class TestDefaults:
"""Verify ABC default implementations work correctly."""
def test_default_tool_schemas_empty(self):
engine = StubEngine()
# StubEngine overrides this, so test the base via super
assert ContextEngine.get_tool_schemas(engine) == []
def test_default_handle_tool_call_returns_error(self):
engine = StubEngine()
result = ContextEngine.handle_tool_call(engine, "unknown", {})
data = json.loads(result)
assert "error" in data
def test_default_get_status(self):
engine = StubEngine()
engine.last_prompt_tokens = 50000
status = engine.get_status()
assert status["last_prompt_tokens"] == 50000
assert status["context_length"] == 200000
assert status["threshold_tokens"] == 100000
assert 0 < status["usage_percent"] <= 100
def test_on_session_reset(self):
engine = StubEngine()
engine.last_prompt_tokens = 999
engine.compression_count = 3
engine.on_session_reset()
assert engine.last_prompt_tokens == 0
assert engine.compression_count == 0
def test_should_compress_preflight_default_false(self):
engine = StubEngine()
assert engine.should_compress_preflight([]) is False
# ---------------------------------------------------------------------------
# StubEngine behavior
# ---------------------------------------------------------------------------
class TestStubEngine:
def test_should_compress(self):
engine = StubEngine(context_length=100000, threshold_pct=0.50)
assert not engine.should_compress(40000)
assert engine.should_compress(50000)
assert engine.should_compress(60000)
def test_compress_tracks_count(self):
engine = StubEngine()
msgs = [{"role": "user", "content": "hello"}]
result = engine.compress(msgs)
assert result == msgs
assert engine._compress_called
assert engine.compression_count == 1
def test_tool_schemas(self):
engine = StubEngine()
schemas = engine.get_tool_schemas()
assert len(schemas) == 1
assert schemas[0]["name"] == "stub_search"
def test_handle_tool_call(self):
engine = StubEngine()
result = engine.handle_tool_call("stub_search", {})
assert json.loads(result)["ok"] is True
assert "stub_search" in engine._tools_called
def test_update_from_response(self):
engine = StubEngine()
engine.update_from_response({"prompt_tokens": 1000, "completion_tokens": 200, "total_tokens": 1200})
assert engine.last_prompt_tokens == 1000
assert engine.last_completion_tokens == 200
# ---------------------------------------------------------------------------
# ContextCompressor session reset via ABC
# ---------------------------------------------------------------------------
class TestCompressorSessionReset:
"""Verify ContextCompressor.on_session_reset() clears all state."""
def test_reset_clears_state(self):
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
c.last_prompt_tokens = 50000
c.compression_count = 3
c._previous_summary = "some old summary"
c._context_probed = True
c._context_probe_persistable = True
c.on_session_reset()
assert c.last_prompt_tokens == 0
assert c.last_completion_tokens == 0
assert c.last_total_tokens == 0
assert c.compression_count == 0
assert c._context_probed is False
assert c._context_probe_persistable is False
assert c._previous_summary is None
# ---------------------------------------------------------------------------
# Plugin slot (PluginManager integration)
# ---------------------------------------------------------------------------
class TestPluginContextEngineSlot:
"""Test register_context_engine on PluginContext."""
def test_register_engine(self):
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest
mgr = PluginManager()
manifest = PluginManifest(name="test-lcm")
ctx = PluginContext(manifest, mgr)
engine = StubEngine()
ctx.register_context_engine(engine)
assert mgr._context_engine is engine
assert mgr._context_engine.name == "stub"
def test_reject_second_engine(self):
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest
mgr = PluginManager()
manifest = PluginManifest(name="test-lcm")
ctx = PluginContext(manifest, mgr)
engine1 = StubEngine()
engine2 = StubEngine()
ctx.register_context_engine(engine1)
ctx.register_context_engine(engine2) # should be rejected
assert mgr._context_engine is engine1
def test_reject_non_engine(self):
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest
mgr = PluginManager()
manifest = PluginManifest(name="test-bad")
ctx = PluginContext(manifest, mgr)
ctx.register_context_engine("not an engine")
assert mgr._context_engine is None
def test_get_plugin_context_engine(self):
from hermes_cli.plugins import PluginManager, get_plugin_context_engine
import hermes_cli.plugins as plugins_mod
# Inject a test manager
old_mgr = plugins_mod._plugin_manager
try:
mgr = PluginManager()
plugins_mod._plugin_manager = mgr
assert get_plugin_context_engine() is None
engine = StubEngine()
mgr._context_engine = engine
assert get_plugin_context_engine() is engine
finally:
plugins_mod._plugin_manager = old_mgr
@@ -0,0 +1,289 @@
"""Regressions for the context-engine host contract.
These tests pin the five generic host-side guarantees that external context
engine plugins (e.g. hermes-lcm) rely on:
1. ``_transition_context_engine_session`` drives the full lifecycle
(on_session_end → on_session_reset → on_session_start → optional
carry_over_new_session_context) and ``reset_session_state`` delegates
to it when callers pass session metadata.
2. ``on_session_start`` receives ``conversation_id`` derived from
``_gateway_session_key`` at agent init time.
3. ``conversation_loop`` forwards canonical cache buckets
(``cache_read_tokens``, ``cache_write_tokens``, ``input_tokens``,
``output_tokens``, ``reasoning_tokens``) to the engine's
``update_from_response``, on top of the legacy aggregate keys.
4. ``_discover_context_engines`` includes plugin-registered engines (not
just repo-shipped engines under ``plugins/context_engine/``).
5. The repo-shipped ``_EngineCollector`` honors ``ctx.register_command``
from a plugin engine's ``register(ctx)`` entry point and routes it
to the global plugin command registry.
"""
from __future__ import annotations
from unittest.mock import MagicMock
from run_agent import AIAgent
def _bare_agent() -> AIAgent:
agent = object.__new__(AIAgent)
agent.session_id = "test-session"
agent.model = "fake-model"
agent.platform = "telegram"
agent._gateway_session_key = "agent:main:telegram:dm:42"
return agent
def test_transition_runs_full_lifecycle_in_order():
"""End → reset → start → carry_over, in that order, when all inputs apply."""
events: list[str] = []
engine = MagicMock()
engine.context_length = 200_000
engine.on_session_end.side_effect = lambda *a, **kw: events.append("on_session_end")
engine.on_session_reset.side_effect = lambda *a, **kw: events.append("on_session_reset")
engine.on_session_start.side_effect = lambda *a, **kw: events.append("on_session_start")
engine.carry_over_new_session_context.side_effect = lambda *a, **kw: events.append("carry_over")
agent = _bare_agent()
agent.context_compressor = engine
agent._transition_context_engine_session(
old_session_id="old-sid",
new_session_id="new-sid",
previous_messages=[{"role": "user", "content": "hi"}],
carry_over_context=True,
)
assert events == [
"on_session_end",
"on_session_reset",
"on_session_start",
"carry_over",
]
def test_transition_passes_conversation_id_from_gateway_session_key():
"""on_session_start receives ``conversation_id`` from ``_gateway_session_key``."""
engine = MagicMock()
engine.context_length = 200_000
captured: dict = {}
engine.on_session_start.side_effect = lambda sid, **kw: captured.update(kw)
agent = _bare_agent()
agent.context_compressor = engine
agent._transition_context_engine_session(
old_session_id="old-sid",
new_session_id="new-sid",
previous_messages=[{"role": "user", "content": "hi"}],
)
assert captured.get("conversation_id") == "agent:main:telegram:dm:42"
assert captured.get("old_session_id") == "old-sid"
assert captured.get("platform") == "telegram"
def test_transition_skips_optional_hooks_when_engine_lacks_them():
"""Engines that don't implement on_session_end/carry_over still work."""
class MinimalEngine:
def __init__(self):
self.context_length = 100_000
self.reset_called = False
self.start_called_with = None
def on_session_reset(self):
self.reset_called = True
def on_session_start(self, sid, **kw):
self.start_called_with = (sid, kw)
engine = MinimalEngine()
agent = _bare_agent()
agent.context_compressor = engine
# Should not raise even though on_session_end / carry_over are missing.
agent._transition_context_engine_session(
old_session_id="old",
new_session_id="new",
previous_messages=[{"role": "user", "content": "hi"}],
carry_over_context=True,
)
assert engine.reset_called is True
assert engine.start_called_with is not None
new_sid, kw = engine.start_called_with
assert new_sid == "new"
assert kw.get("old_session_id") == "old"
def test_reset_session_state_delegates_to_transition_when_args_provided():
"""``reset_session_state(previous_messages=..., old_session_id=...)`` fires full lifecycle."""
engine = MagicMock()
engine.context_length = 100_000
agent = _bare_agent()
agent.context_compressor = engine
agent.reset_session_state(
previous_messages=[{"role": "user", "content": "hi"}],
old_session_id="old-sid",
)
assert engine.on_session_end.called
assert engine.on_session_reset.called
assert engine.on_session_start.called
# No carry_over_context, so carry_over hook NOT called.
assert not engine.carry_over_new_session_context.called
def test_reset_session_state_default_call_only_resets():
"""Bare ``reset_session_state()`` still only resets the engine (no end/start)."""
engine = MagicMock()
engine.context_length = 100_000
agent = _bare_agent()
agent.context_compressor = engine
agent.reset_session_state()
assert engine.on_session_reset.called
assert not engine.on_session_end.called
assert not engine.on_session_start.called
def test_update_from_response_forwards_canonical_cache_buckets():
"""conversation_loop passes cache_read/write/reasoning tokens to engine."""
# Test the contract directly: a usage_dict built from CanonicalUsage must
# contain the canonical buckets in addition to the legacy keys. We don't
# spin up the full conversation loop; we just verify the dict shape.
from agent.usage_pricing import CanonicalUsage
canonical = CanonicalUsage(
input_tokens=1000,
output_tokens=500,
cache_read_tokens=800,
cache_write_tokens=200,
reasoning_tokens=50,
)
usage_dict = {
"prompt_tokens": canonical.prompt_tokens,
"completion_tokens": canonical.output_tokens,
"total_tokens": canonical.total_tokens,
"input_tokens": canonical.input_tokens,
"output_tokens": canonical.output_tokens,
"cache_read_tokens": canonical.cache_read_tokens,
"cache_write_tokens": canonical.cache_write_tokens,
"reasoning_tokens": canonical.reasoning_tokens,
}
# Legacy keys present
assert usage_dict["prompt_tokens"] == canonical.prompt_tokens
assert usage_dict["completion_tokens"] == 500
assert usage_dict["total_tokens"] == canonical.total_tokens
# Canonical cache + reasoning buckets present
assert usage_dict["cache_read_tokens"] == 800
assert usage_dict["cache_write_tokens"] == 200
assert usage_dict["reasoning_tokens"] == 50
assert usage_dict["input_tokens"] == 1000
assert usage_dict["output_tokens"] == 500
def test_discover_context_engines_includes_plugin_registered_engines(monkeypatch):
"""Plugin-registered context engines appear in the ``hermes plugins`` picker."""
from hermes_cli import plugins_cmd
fake_repo = lambda: [("compressor", "built-in", True)]
class FakePluginEngine:
name = "lcm"
monkeypatch.setattr(
"plugins.context_engine.discover_context_engines",
fake_repo,
)
monkeypatch.setattr(
"hermes_cli.plugins.discover_plugins",
lambda *_a, **_kw: None,
)
monkeypatch.setattr(
"hermes_cli.plugins.get_plugin_context_engine",
lambda: FakePluginEngine(),
)
engines = plugins_cmd._discover_context_engines()
names = [n for n, _desc in engines]
assert "compressor" in names
assert "lcm" in names
def test_discover_context_engines_dedupes_by_name(monkeypatch):
"""Repo-shipped engine wins when name collides with a plugin-registered one."""
from hermes_cli import plugins_cmd
class FakePluginEngine:
name = "compressor" # same name as repo-shipped
monkeypatch.setattr(
"plugins.context_engine.discover_context_engines",
lambda: [("compressor", "built-in compressor", True)],
)
monkeypatch.setattr(
"hermes_cli.plugins.discover_plugins",
lambda *_a, **_kw: None,
)
monkeypatch.setattr(
"hermes_cli.plugins.get_plugin_context_engine",
lambda: FakePluginEngine(),
)
engines = plugins_cmd._discover_context_engines()
# Only one entry — the repo-shipped one. Description is preserved.
assert engines == [("compressor", "built-in compressor")]
def test_engine_collector_forwards_register_command_to_plugin_manager():
"""A plugin context engine can register a slash command via ``ctx.register_command``."""
from plugins.context_engine import _EngineCollector
from hermes_cli.plugins import get_plugin_manager
handler = lambda raw_args: f"echo: {raw_args}"
collector = _EngineCollector(engine_name="my-lcm")
collector.register_command(
"my-lcm-test-cmd",
handler,
description="test command from a context engine",
args_hint="<msg>",
)
manager = get_plugin_manager()
try:
assert "my-lcm-test-cmd" in manager._plugin_commands
entry = manager._plugin_commands["my-lcm-test-cmd"]
assert entry["handler"] is handler
assert entry["args_hint"] == "<msg>"
assert entry["plugin"] == "context-engine:my-lcm"
finally:
# Clean up so we don't leak the registration across tests.
manager._plugin_commands.pop("my-lcm-test-cmd", None)
def test_engine_collector_rejects_builtin_command_conflicts():
"""Context engine cannot shadow built-in slash commands like /help."""
from plugins.context_engine import _EngineCollector
from hermes_cli.plugins import get_plugin_manager
collector = _EngineCollector(engine_name="my-lcm")
collector.register_command("help", lambda *_: "shadow")
manager = get_plugin_manager()
# Must NOT have overwritten / registered against built-in /help.
assert "help" not in manager._plugin_commands or \
manager._plugin_commands["help"].get("plugin") != "context-engine:my-lcm"
+336
View File
@@ -0,0 +1,336 @@
from __future__ import annotations
import asyncio
import subprocess
from pathlib import Path
from unittest.mock import patch
import pytest
def _git(cwd: Path, *args: str) -> str:
result = subprocess.run(
["git", *args],
cwd=cwd,
check=True,
capture_output=True,
text=True,
)
return result.stdout.strip()
@pytest.fixture
def sample_repo(tmp_path: Path) -> Path:
repo = tmp_path / "repo"
repo.mkdir()
_git(repo, "init")
_git(repo, "config", "user.name", "Hermes Tests")
_git(repo, "config", "user.email", "tests@example.com")
(repo / "src").mkdir()
(repo / "src" / "main.py").write_text(
"def alpha():\n"
" return 'a'\n\n"
"def beta():\n"
" return 'b'\n",
encoding="utf-8",
)
(repo / "src" / "helper.py").write_text("VALUE = 1\n", encoding="utf-8")
(repo / "README.md").write_text("# Demo\n", encoding="utf-8")
(repo / "blob.bin").write_bytes(b"\x00\x01\x02binary")
_git(repo, "add", ".")
_git(repo, "commit", "-m", "initial")
(repo / "src" / "main.py").write_text(
"def alpha():\n"
" return 'changed'\n\n"
"def beta():\n"
" return 'b'\n",
encoding="utf-8",
)
(repo / "src" / "helper.py").write_text("VALUE = 2\n", encoding="utf-8")
_git(repo, "add", "src/helper.py")
return repo
def test_parse_typed_references_ignores_emails_and_handles():
from agent.context_references import parse_context_references
message = (
"email me at user@example.com and ping @teammate "
"but include @file:src/main.py:1-2 plus @diff and @git:2 "
"and @url:https://example.com/docs"
)
refs = parse_context_references(message)
assert [ref.kind for ref in refs] == ["file", "diff", "git", "url"]
assert refs[0].target == "src/main.py"
assert refs[0].line_start == 1
assert refs[0].line_end == 2
assert refs[2].target == "2"
def test_parse_references_strips_trailing_punctuation():
from agent.context_references import parse_context_references
refs = parse_context_references(
"review @file:README.md, then see (@url:https://example.com/docs)."
)
assert [ref.kind for ref in refs] == ["file", "url"]
assert refs[0].target == "README.md"
assert refs[1].target == "https://example.com/docs"
def test_parse_quoted_references_with_spaces_and_preserve_unquoted_ranges():
from agent.context_references import parse_context_references
refs = parse_context_references(
'review @file:"C:\\Users\\Simba\\My Project\\main.py":7-9 '
'and @folder:"docs and specs" plus @file:src/main.py:1-2'
)
assert [ref.kind for ref in refs] == ["file", "folder", "file"]
assert refs[0].target == r"C:\Users\Simba\My Project\main.py"
assert refs[0].line_start == 7
assert refs[0].line_end == 9
assert refs[1].target == "docs and specs"
assert refs[2].target == "src/main.py"
assert refs[2].line_start == 1
assert refs[2].line_end == 2
def test_expand_file_range_and_folder_listing(sample_repo: Path):
from agent.context_references import preprocess_context_references
result = preprocess_context_references(
"Review @file:src/main.py:1-2 and @folder:src/",
cwd=sample_repo,
context_length=100_000,
)
assert result.expanded
assert "Review and" in result.message
assert "Review @file:src/main.py:1-2" not in result.message
assert "--- Attached Context ---" in result.message
assert "def alpha():" in result.message
assert "return 'changed'" in result.message
assert "def beta():" not in result.message
assert "src/" in result.message
assert "main.py" in result.message
assert "helper.py" in result.message
assert result.injected_tokens > 0
assert not result.warnings
def test_folder_listing_falls_back_when_rg_is_blocked(sample_repo: Path):
from agent.context_references import preprocess_context_references
real_run = subprocess.run
def blocked_rg(*args, **kwargs):
cmd = args[0] if args else kwargs.get("args")
if isinstance(cmd, list) and cmd and cmd[0] == "rg":
raise PermissionError("rg blocked by policy")
return real_run(*args, **kwargs)
with patch("agent.context_references.subprocess.run", side_effect=blocked_rg):
result = preprocess_context_references(
"Review @folder:src/",
cwd=sample_repo,
context_length=100_000,
)
assert result.expanded
assert "src/" in result.message
assert "main.py" in result.message
assert "helper.py" in result.message
assert not result.warnings
def test_expand_quoted_file_reference_with_spaces(tmp_path: Path):
from agent.context_references import preprocess_context_references
workspace = tmp_path / "repo"
folder = workspace / "docs and specs"
folder.mkdir(parents=True)
file_path = folder / "release notes.txt"
file_path.write_text("line 1\nline 2\nline 3\n", encoding="utf-8")
result = preprocess_context_references(
'Review @file:"docs and specs/release notes.txt":2-3',
cwd=workspace,
context_length=100_000,
)
assert result.expanded
assert result.message.startswith("Review")
assert "line 1" not in result.message
assert "line 2" in result.message
assert "line 3" in result.message
assert "release notes.txt" in result.message
assert not result.warnings
def test_expand_git_diff_staged_and_log(sample_repo: Path):
from agent.context_references import preprocess_context_references
result = preprocess_context_references(
"Inspect @diff and @staged and @git:1",
cwd=sample_repo,
context_length=100_000,
)
assert result.expanded
assert "git diff" in result.message
assert "git diff --staged" in result.message
assert "git log -1 -p" in result.message
assert "initial" in result.message
assert "return 'changed'" in result.message
assert "VALUE = 2" in result.message
def test_binary_and_missing_files_become_warnings(sample_repo: Path):
from agent.context_references import preprocess_context_references
result = preprocess_context_references(
"Check @file:blob.bin and @file:nope.txt",
cwd=sample_repo,
context_length=100_000,
)
assert result.expanded
assert len(result.warnings) == 2
assert "binary" in result.message.lower()
assert "not found" in result.message.lower()
def test_soft_budget_warns_and_hard_budget_refuses(sample_repo: Path):
from agent.context_references import preprocess_context_references
soft = preprocess_context_references(
"Check @file:src/main.py",
cwd=sample_repo,
context_length=100,
)
assert soft.expanded
assert any("25%" in warning for warning in soft.warnings)
hard = preprocess_context_references(
"Check @file:src/main.py and @file:README.md",
cwd=sample_repo,
context_length=20,
)
assert not hard.expanded
assert hard.blocked
assert "@file:src/main.py" in hard.message
assert any("50%" in warning for warning in hard.warnings)
@pytest.mark.asyncio
async def test_async_url_expansion_uses_fetcher(sample_repo: Path):
from agent.context_references import preprocess_context_references_async
async def fake_fetch(url: str) -> str:
assert url == "https://example.com/spec"
return "# Spec\n\nImportant details."
result = await preprocess_context_references_async(
"Use @url:https://example.com/spec",
cwd=sample_repo,
context_length=100_000,
url_fetcher=fake_fetch,
)
assert result.expanded
assert "Important details." in result.message
assert result.injected_tokens > 0
def test_sync_url_expansion_uses_async_fetcher(sample_repo: Path):
from agent.context_references import preprocess_context_references
async def fake_fetch(url: str) -> str:
await asyncio.sleep(0)
return f"Content for {url}"
result = preprocess_context_references(
"Use @url:https://example.com/spec",
cwd=sample_repo,
context_length=100_000,
url_fetcher=fake_fetch,
)
assert result.expanded
assert "Content for https://example.com/spec" in result.message
def test_restricts_paths_to_allowed_root(tmp_path: Path):
from agent.context_references import preprocess_context_references
workspace = tmp_path / "workspace"
workspace.mkdir()
(workspace / "notes.txt").write_text("inside\n", encoding="utf-8")
secret = tmp_path / "secret.txt"
secret.write_text("outside\n", encoding="utf-8")
result = preprocess_context_references(
"read @file:../secret.txt and @file:notes.txt",
cwd=workspace,
context_length=100_000,
allowed_root=workspace,
)
assert result.expanded
assert "```\noutside\n```" not in result.message
assert "inside" in result.message
assert any("outside the allowed workspace" in warning for warning in result.warnings)
def test_defaults_allowed_root_to_cwd(tmp_path: Path):
from agent.context_references import preprocess_context_references
workspace = tmp_path / "workspace"
workspace.mkdir()
secret = tmp_path / "secret.txt"
secret.write_text("outside\n", encoding="utf-8")
result = preprocess_context_references(
f"read @file:{secret}",
cwd=workspace,
context_length=100_000,
)
assert result.expanded
assert "```\noutside\n```" not in result.message
assert any("outside the allowed workspace" in warning for warning in result.warnings)
@pytest.mark.asyncio
async def test_blocks_sensitive_home_and_hermes_paths(tmp_path: Path, monkeypatch):
from agent.context_references import preprocess_context_references_async
monkeypatch.setenv("HOME", str(tmp_path))
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
hermes_env = tmp_path / ".hermes" / ".env"
hermes_env.parent.mkdir(parents=True)
hermes_env.write_text("API_KEY=super-secret\n", encoding="utf-8")
ssh_key = tmp_path / ".ssh" / "id_rsa"
ssh_key.parent.mkdir(parents=True)
ssh_key.write_text("PRIVATE-KEY\n", encoding="utf-8")
result = await preprocess_context_references_async(
"read @file:.hermes/.env and @file:.ssh/id_rsa",
cwd=tmp_path,
allowed_root=tmp_path,
context_length=100_000,
)
assert result.expanded
assert "API_KEY=super-secret" not in result.message
assert "PRIVATE-KEY" not in result.message
assert any("sensitive credential" in warning for warning in result.warnings)
+207
View File
@@ -0,0 +1,207 @@
"""Focused regressions for the Copilot ACP shim safety layer."""
from __future__ import annotations
import io
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from agent.copilot_acp_client import CopilotACPClient
class _FakeProcess:
def __init__(self) -> None:
self.stdin = io.StringIO()
class CopilotACPClientSafetyTests(unittest.TestCase):
def setUp(self) -> None:
self.client = CopilotACPClient(acp_cwd="/tmp")
def _dispatch(self, message: dict, *, cwd: str) -> dict:
process = _FakeProcess()
handled = self.client._handle_server_message(
message,
process=process,
cwd=cwd,
text_parts=[],
reasoning_parts=[],
)
self.assertTrue(handled)
payload = process.stdin.getvalue().strip()
self.assertTrue(payload)
return json.loads(payload)
def test_request_permission_is_not_auto_allowed(self) -> None:
response = self._dispatch(
{
"jsonrpc": "2.0",
"id": 1,
"method": "session/request_permission",
"params": {},
},
cwd="/tmp",
)
outcome = (((response.get("result") or {}).get("outcome") or {}).get("outcome"))
self.assertEqual(outcome, "cancelled")
def test_read_text_file_blocks_internal_hermes_hub_files(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
home = Path(tmpdir) / "home"
blocked = home / ".hermes" / "skills" / ".hub" / "index-cache" / "entry.json"
blocked.parent.mkdir(parents=True, exist_ok=True)
blocked.write_text('{"token":"sk-test-secret-1234567890"}')
with patch.dict(
os.environ,
{"HOME": str(home), "HERMES_HOME": str(home / ".hermes")},
clear=False,
):
response = self._dispatch(
{
"jsonrpc": "2.0",
"id": 2,
"method": "fs/read_text_file",
"params": {"path": str(blocked)},
},
cwd=str(home),
)
self.assertIn("error", response)
def test_read_text_file_redacts_sensitive_content(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
secret_file = root / "config.env"
secret_file.write_text("OPENAI_API_KEY=sk-proj-abc123def456ghi789jkl012")
# agent.redact snapshots HERMES_REDACT_SECRETS at import time into
# _REDACT_ENABLED, so patching os.environ is a no-op. Flip the
# module-level constant directly for the duration of the call.
with patch("agent.redact._REDACT_ENABLED", True):
response = self._dispatch(
{
"jsonrpc": "2.0",
"id": 3,
"method": "fs/read_text_file",
"params": {"path": str(secret_file)},
},
cwd=str(root),
)
content = ((response.get("result") or {}).get("content") or "")
self.assertNotIn("abc123def456", content)
self.assertIn("OPENAI_API_KEY=", content)
def test_write_text_file_reuses_write_denylist(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
home = Path(tmpdir) / "home"
target = home / ".ssh" / "id_rsa"
target.parent.mkdir(parents=True, exist_ok=True)
with patch("agent.copilot_acp_client.is_write_denied", return_value=True, create=True):
response = self._dispatch(
{
"jsonrpc": "2.0",
"id": 4,
"method": "fs/write_text_file",
"params": {
"path": str(target),
"content": "fake-private-key",
},
},
cwd=str(home),
)
self.assertIn("error", response)
self.assertFalse(target.exists())
def test_write_text_file_respects_safe_root(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
safe_root = root / "workspace"
safe_root.mkdir()
outside = root / "outside.txt"
with patch.dict(os.environ, {"HERMES_WRITE_SAFE_ROOT": str(safe_root)}, clear=False):
response = self._dispatch(
{
"jsonrpc": "2.0",
"id": 5,
"method": "fs/write_text_file",
"params": {
"path": str(outside),
"content": "should-not-write",
},
},
cwd=str(root),
)
self.assertIn("error", response)
self.assertFalse(outside.exists())
if __name__ == "__main__":
unittest.main()
# ── HOME env propagation tests (from PR #11285) ─────────────────────
from unittest.mock import patch as _patch
import pytest
def _make_home_client(tmp_path):
return CopilotACPClient(
api_key="copilot-acp",
base_url="acp://copilot",
acp_command="copilot",
acp_args=["--acp", "--stdio"],
acp_cwd=str(tmp_path),
)
def _fake_popen_capture(captured):
def _fake(cmd, **kwargs):
captured["cmd"] = cmd
captured["kwargs"] = kwargs
raise FileNotFoundError("copilot not found")
return _fake
def test_run_prompt_prefers_profile_home_when_available(monkeypatch, tmp_path):
hermes_home = tmp_path / "hermes"
profile_home = hermes_home / "home"
profile_home.mkdir(parents=True)
monkeypatch.delenv("HOME", raising=False)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
captured = {}
client = _make_home_client(tmp_path)
with _patch("agent.copilot_acp_client.subprocess.Popen", side_effect=_fake_popen_capture(captured)):
with pytest.raises(RuntimeError, match="Could not start Copilot ACP command"):
client._run_prompt("hello", timeout_seconds=1)
assert captured["kwargs"]["env"]["HOME"] == str(profile_home)
def test_run_prompt_passes_home_when_parent_env_is_clean(monkeypatch, tmp_path):
monkeypatch.delenv("HOME", raising=False)
monkeypatch.delenv("HERMES_HOME", raising=False)
captured = {}
client = _make_home_client(tmp_path)
with _patch("agent.copilot_acp_client.subprocess.Popen", side_effect=_fake_popen_capture(captured)):
with pytest.raises(RuntimeError, match="Could not start Copilot ACP command"):
client._run_prompt("hello", timeout_seconds=1)
assert "env" in captured["kwargs"]
assert captured["kwargs"]["env"]["HOME"]
@@ -0,0 +1,77 @@
"""Tests for gh-copilot CLI deprecation detection and GitHub Models Azure URL mapping."""
import pytest
from agent.copilot_acp_client import _is_gh_copilot_deprecation_message
class TestDeprecationPatternDetection:
"""Verify that stderr from the deprecated `gh copilot` extension is caught
without false-positiving on the new `@github/copilot` CLI."""
_REAL_DEPRECATION_STDERR = (
"The gh-copilot extension has been deprecated in favor of the newer "
"GitHub Copilot CLI.\nFor more information, visit:\n"
"- Copilot CLI: https://github.com/github/copilot-cli\n"
"- Deprecation announcement: https://github.blog/changelog/"
"2025-09-25-upcoming-deprecation-of-gh-copilot-cli-extension\n"
"No commands will be executed."
)
def test_real_deprecation_message_matches(self):
assert _is_gh_copilot_deprecation_message(self._REAL_DEPRECATION_STDERR)
@pytest.mark.parametrize(
"stderr_text",
[
# The deprecation banner uses both halves of the fingerprint.
"The gh-copilot extension has been deprecated.",
"gh-copilot: no commands will be executed.",
# Mixed casing — match is case-insensitive.
"The GH-Copilot Extension HAS BEEN DEPRECATED.",
],
)
def test_genuine_deprecation_variants_match(self, stderr_text: str):
assert _is_gh_copilot_deprecation_message(stderr_text)
@pytest.mark.parametrize(
"stderr_text",
[
# Generic errors — no fingerprint at all.
"Error: connection refused",
"",
# The NEW @github/copilot CLI's repo is github.com/github/copilot-cli.
# Its stderr can legitimately mention "copilot-cli" or "deprecation"
# in unrelated contexts; neither alone should trip the detector.
"copilot-cli: failed to authenticate with the API",
"warning: the --foo flag is scheduled for deprecation in v3",
"See https://github.com/github/copilot-cli/issues for support",
# Half the fingerprint without the other half.
"gh-copilot: command not found",
"extension has been deprecated (some other extension)",
],
)
def test_does_not_false_positive(self, stderr_text: str):
assert not _is_gh_copilot_deprecation_message(stderr_text)
class TestGitHubModelsAzureUrl:
"""Verify that the Azure GitHub Models URL is recognised."""
def test_url_to_provider_contains_azure_models(self):
from agent.model_metadata import _URL_TO_PROVIDER
# Maps to the canonical "copilot" provider (same convention as the
# other GitHub-family entries) — not the "github-models" alias.
assert _URL_TO_PROVIDER.get("models.inference.ai.azure.com") == "copilot"
def test_is_github_models_base_url_recognises_azure(self):
from hermes_cli.models import _is_github_models_base_url
assert _is_github_models_base_url("https://models.inference.ai.azure.com")
assert _is_github_models_base_url("https://models.inference.ai.azure.com/v1/chat")
def test_is_github_models_base_url_still_recognises_github_ai(self):
from hermes_cli.models import _is_github_models_base_url
assert _is_github_models_base_url("https://models.github.ai/inference")
File diff suppressed because it is too large Load Diff
+234
View File
@@ -0,0 +1,234 @@
"""Tests for credential pool preservation through turn config and 429 recovery.
Covers:
1. CLI _resolve_turn_agent_config passes credential_pool to runtime dict
2. Gateway _resolve_turn_agent_config passes credential_pool to runtime dict
3. Eager fallback deferred when credential pool has credentials
4. Eager fallback fires when no credential pool exists
5. Full 429 rotation cycle: retry-same → rotate → exhaust → fallback
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
# ---------------------------------------------------------------------------
# 1. CLI _resolve_turn_agent_config includes credential_pool
# ---------------------------------------------------------------------------
class TestCliTurnRoutePool:
def test_resolve_turn_includes_pool(self):
"""CLI's _resolve_turn_agent_config must pass credential_pool in runtime."""
fake_pool = MagicMock(name="FakePool")
shell = SimpleNamespace(
model="gpt-5.4",
api_key="sk-test",
base_url=None,
provider="openai-codex",
api_mode="codex_responses",
acp_command=None,
acp_args=[],
_credential_pool=fake_pool,
service_tier=None,
)
from cli import HermesCLI
bound = HermesCLI._resolve_turn_agent_config.__get__(shell)
route = bound("test message")
assert route["runtime"]["credential_pool"] is fake_pool
# ---------------------------------------------------------------------------
# 2. Gateway _resolve_turn_agent_config includes credential_pool
# ---------------------------------------------------------------------------
class TestGatewayTurnRoutePool:
def test_resolve_turn_includes_pool(self):
"""Gateway's _resolve_turn_agent_config must pass credential_pool."""
from gateway.run import GatewayRunner
fake_pool = MagicMock(name="FakePool")
runner = SimpleNamespace(_service_tier=None)
runtime_kwargs = {
"api_key": "***",
"base_url": None,
"provider": "openai-codex",
"api_mode": "codex_responses",
"command": None,
"args": [],
"credential_pool": fake_pool,
}
bound = GatewayRunner._resolve_turn_agent_config.__get__(runner)
route = bound("test message", "gpt-5.4", runtime_kwargs)
assert route["runtime"]["credential_pool"] is fake_pool
# ---------------------------------------------------------------------------
# 3 & 4. Eager fallback deferred/fires based on credential pool
# ---------------------------------------------------------------------------
class TestEagerFallbackWithPool:
"""Test the eager fallback guard in run_agent.py's error handling loop."""
def _make_agent(self, has_pool=True, pool_has_creds=True, has_fallback=True):
"""Create a minimal AIAgent mock with the fields needed."""
from run_agent import AIAgent
with patch.object(AIAgent, "__init__", lambda self, **kw: None):
agent = AIAgent()
agent._credential_pool = None
if has_pool:
pool = MagicMock()
pool.has_available.return_value = pool_has_creds
agent._credential_pool = pool
agent._fallback_chain = [{"model": "fallback/model"}] if has_fallback else []
agent._fallback_index = 0
agent._try_activate_fallback = MagicMock(return_value=True)
agent._emit_status = MagicMock()
return agent
def test_eager_fallback_deferred_when_pool_has_credentials(self):
"""429 with active pool should NOT trigger eager fallback."""
agent = self._make_agent(has_pool=True, pool_has_creds=True, has_fallback=True)
# Simulate the check from run_agent.py lines 7180-7191
is_rate_limited = True
if is_rate_limited and agent._fallback_index < len(agent._fallback_chain):
pool = agent._credential_pool
pool_may_recover = pool is not None and pool.has_available()
if not pool_may_recover:
agent._try_activate_fallback()
agent._try_activate_fallback.assert_not_called()
def test_eager_fallback_fires_when_no_pool(self):
"""429 without pool should trigger eager fallback."""
agent = self._make_agent(has_pool=False, has_fallback=True)
is_rate_limited = True
if is_rate_limited and agent._fallback_index < len(agent._fallback_chain):
pool = agent._credential_pool
pool_may_recover = pool is not None and pool.has_available()
if not pool_may_recover:
agent._try_activate_fallback()
agent._try_activate_fallback.assert_called_once()
def test_eager_fallback_fires_when_pool_exhausted(self):
"""429 with exhausted pool should trigger eager fallback."""
agent = self._make_agent(has_pool=True, pool_has_creds=False, has_fallback=True)
is_rate_limited = True
if is_rate_limited and agent._fallback_index < len(agent._fallback_chain):
pool = agent._credential_pool
pool_may_recover = pool is not None and pool.has_available()
if not pool_may_recover:
agent._try_activate_fallback()
agent._try_activate_fallback.assert_called_once()
# ---------------------------------------------------------------------------
# 5. Full 429 rotation cycle via _recover_with_credential_pool
# ---------------------------------------------------------------------------
class TestPoolRotationCycle:
"""Verify the retry-same → rotate → exhaust flow in _recover_with_credential_pool."""
def _make_agent_with_pool(self, pool_entries=3):
from run_agent import AIAgent
with patch.object(AIAgent, "__init__", lambda self, **kw: None):
agent = AIAgent()
entries = []
for i in range(pool_entries):
e = MagicMock(name=f"entry_{i}")
e.id = f"cred-{i}"
entries.append(e)
pool = MagicMock()
pool.has_credentials.return_value = True
# mark_exhausted_and_rotate returns next entry until exhausted
self._rotation_index = 0
def rotate(status_code=None, error_context=None):
self._rotation_index += 1
if self._rotation_index < pool_entries:
return entries[self._rotation_index]
pool.has_credentials.return_value = False
return None
pool.mark_exhausted_and_rotate = MagicMock(side_effect=rotate)
agent._credential_pool = pool
agent._swap_credential = MagicMock()
agent.log_prefix = ""
return agent, pool, entries
def test_first_429_sets_retry_flag_no_rotation(self):
"""First 429 should just set has_retried_429=True, no rotation."""
agent, pool, _ = self._make_agent_with_pool(3)
recovered, has_retried = agent._recover_with_credential_pool(
status_code=429, has_retried_429=False
)
assert recovered is False
assert has_retried is True
pool.mark_exhausted_and_rotate.assert_not_called()
def test_second_429_rotates_to_next(self):
"""Second consecutive 429 should rotate to next credential."""
agent, pool, entries = self._make_agent_with_pool(3)
recovered, has_retried = agent._recover_with_credential_pool(
status_code=429, has_retried_429=True
)
assert recovered is True
assert has_retried is False # reset after rotation
pool.mark_exhausted_and_rotate.assert_called_once_with(status_code=429, error_context=None)
agent._swap_credential.assert_called_once_with(entries[1])
def test_pool_exhaustion_returns_false(self):
"""When all credentials exhausted, recovery should return False."""
agent, pool, _ = self._make_agent_with_pool(1)
# First 429 sets flag
_, has_retried = agent._recover_with_credential_pool(
status_code=429, has_retried_429=False
)
assert has_retried is True
# Second 429 tries to rotate but pool is exhausted (only 1 entry)
recovered, _ = agent._recover_with_credential_pool(
status_code=429, has_retried_429=True
)
assert recovered is False
def test_402_immediate_rotation(self):
"""402 (billing) should immediately rotate, no retry-first."""
agent, pool, entries = self._make_agent_with_pool(3)
recovered, has_retried = agent._recover_with_credential_pool(
status_code=402, has_retried_429=False
)
assert recovered is True
assert has_retried is False
pool.mark_exhausted_and_rotate.assert_called_once_with(status_code=402, error_context=None)
def test_no_pool_returns_false(self):
"""No pool should return (False, unchanged)."""
from run_agent import AIAgent
with patch.object(AIAgent, "__init__", lambda self, **kw: None):
agent = AIAgent()
agent._credential_pool = None
recovered, has_retried = agent._recover_with_credential_pool(
status_code=429, has_retried_429=False
)
assert recovered is False
assert has_retried is False
+183
View File
@@ -0,0 +1,183 @@
"""Tests for cross-loop client cache isolation fix (#2681).
Verifies that _get_cached_client() returns different AsyncOpenAI clients
when called from different event loops, preventing the httpx deadlock
that occurs when a cached async client bound to loop A is reused on loop B.
This test file is self-contained and does not import the full tool chain,
so it can run without optional dependencies like firecrawl.
"""
import asyncio
import threading
from unittest.mock import patch, MagicMock
import pytest
# ---------------------------------------------------------------------------
# Minimal stubs so we can import _get_cached_client without the full tree
# ---------------------------------------------------------------------------
def _stub_resolve_provider_client(provider, model, async_mode, **kw):
"""Return a unique mock client each time, simulating AsyncOpenAI creation."""
client = MagicMock(name=f"client-{provider}-async={async_mode}")
client.api_key = "test"
client.base_url = kw.get("explicit_base_url", "http://localhost:8081/v1")
return client, model or "test-model"
@pytest.fixture(autouse=True)
def _clean_client_cache():
"""Clear the client cache before each test."""
# We need to patch before importing
with patch.dict("sys.modules", {}):
pass
# Import and clear
import agent.auxiliary_client as ac
ac._client_cache.clear()
yield
ac._client_cache.clear()
class TestCrossLoopCacheIsolation:
"""Verify async clients are cached per-event-loop, not globally."""
def test_same_loop_reuses_client(self):
"""Within a single event loop, the same client should be returned."""
from agent.auxiliary_client import _get_cached_client
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
client1, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
client2, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
assert client1 is client2, (
"Same loop should return the same cached client"
)
loop.close()
def test_different_loops_get_different_clients(self):
"""Different event loops must get separate client instances."""
from agent.auxiliary_client import _get_cached_client
results = {}
def _get_client_on_new_loop(name):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
client, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
results[name] = (id(client), id(loop))
# Don't close loop — simulates real usage where loops persist
t1 = threading.Thread(target=_get_client_on_new_loop, args=("a",))
t2 = threading.Thread(target=_get_client_on_new_loop, args=("b",))
t1.start(); t1.join()
t2.start(); t2.join()
client_id_a, loop_id_a = results["a"]
client_id_b, loop_id_b = results["b"]
assert loop_id_a != loop_id_b, "Test setup error: same loop on both threads"
assert client_id_a != client_id_b, (
"Different event loops got the SAME cached client — this causes "
"httpx cross-loop deadlocks in gateway mode (#2681)"
)
def test_sync_clients_not_affected(self):
"""Sync clients (async_mode=False) should still be cached globally,
since httpx.Client (sync) doesn't bind to an event loop."""
from agent.auxiliary_client import _get_cached_client
results = {}
def _get_sync_client(name):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
client, _ = _get_cached_client("custom", "m1", async_mode=False,
base_url="http://localhost:8081/v1")
results[name] = id(client)
t1 = threading.Thread(target=_get_sync_client, args=("a",))
t2 = threading.Thread(target=_get_sync_client, args=("b",))
t1.start(); t1.join()
t2.start(); t2.join()
assert results["a"] == results["b"], (
"Sync clients should be shared across threads (no loop binding)"
)
def test_gateway_simulation_no_deadlock(self):
"""Simulate gateway mode: _run_async spawns a thread with asyncio.run(),
which creates a new loop. The cached client must be created on THAT loop,
not reused from a different one."""
from agent.auxiliary_client import _get_cached_client
# Simulate: first call on "gateway loop"
gateway_loop = asyncio.new_event_loop()
asyncio.set_event_loop(gateway_loop)
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
gateway_client, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
# Simulate: _run_async spawns a thread with asyncio.run()
worker_client_id = [None]
def _worker():
async def _inner():
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
client, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
worker_client_id[0] = id(client)
asyncio.run(_inner())
t = threading.Thread(target=_worker)
t.start()
t.join()
assert worker_client_id[0] != id(gateway_client), (
"Worker thread (asyncio.run) got the gateway's cached client — "
"this is the exact cross-loop scenario that causes httpx deadlocks. "
"The cache key must include the event loop identity (#2681)"
)
gateway_loop.close()
def test_closed_loop_client_discarded(self):
"""A cached client whose loop has closed should be replaced."""
from agent.auxiliary_client import _get_cached_client
loop1 = asyncio.new_event_loop()
asyncio.set_event_loop(loop1)
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
client1, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
loop1.close()
# New loop on same thread
loop2 = asyncio.new_event_loop()
asyncio.set_event_loop(loop2)
with patch("agent.auxiliary_client.resolve_provider_client",
side_effect=_stub_resolve_provider_client):
client2, _ = _get_cached_client("custom", "m1", async_mode=True,
base_url="http://localhost:8081/v1")
assert client1 is not client2, (
"Client from closed loop should not be reused"
)
loop2.close()
+885
View File
@@ -0,0 +1,885 @@
"""Tests for agent/curator.py — orchestrator, idle gating, state transitions.
LLM spawning is never exercised here — `_run_llm_review` is monkeypatched so
tests run fully offline and the curator module doesn't need real credentials.
"""
from __future__ import annotations
import importlib
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
@pytest.fixture
def curator_env(tmp_path, monkeypatch):
"""Isolated HERMES_HOME + freshly reloaded curator + skill_usage modules."""
home = tmp_path / ".hermes"
(home / "skills").mkdir(parents=True)
monkeypatch.setattr(Path, "home", lambda: tmp_path)
monkeypatch.setenv("HERMES_HOME", str(home))
import tools.skill_usage as usage
importlib.reload(usage)
import agent.curator as curator
importlib.reload(curator)
# Neutralize the real LLM pass by default — tests opt in per-case.
monkeypatch.setattr(curator, "_run_llm_review", lambda prompt: "llm-stub")
# Default: no config file → curator defaults. Tests can override.
monkeypatch.setattr(curator, "_load_config", lambda: {})
return {"home": home, "curator": curator, "usage": usage}
def _write_skill(skills_dir: Path, name: str):
d = skills_dir / name
d.mkdir(parents=True, exist_ok=True)
(d / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: x\n---\n", encoding="utf-8",
)
return d
# ---------------------------------------------------------------------------
# Config gates
# ---------------------------------------------------------------------------
def test_curator_enabled_default_true(curator_env):
assert curator_env["curator"].is_enabled() is True
def test_curator_disabled_via_config(curator_env, monkeypatch):
c = curator_env["curator"]
monkeypatch.setattr(c, "_load_config", lambda: {"enabled": False})
assert c.is_enabled() is False
assert c.should_run_now() is False
def test_curator_defaults(curator_env):
c = curator_env["curator"]
assert c.get_interval_hours() == 24 * 7 # 7 days
assert c.get_min_idle_hours() == 2
assert c.get_stale_after_days() == 30
assert c.get_archive_after_days() == 90
def test_curator_config_overrides(curator_env, monkeypatch):
c = curator_env["curator"]
monkeypatch.setattr(c, "_load_config", lambda: {
"interval_hours": 12,
"min_idle_hours": 0.5,
"stale_after_days": 7,
"archive_after_days": 60,
})
assert c.get_interval_hours() == 12
assert c.get_min_idle_hours() == 0.5
assert c.get_stale_after_days() == 7
assert c.get_archive_after_days() == 60
# ---------------------------------------------------------------------------
# should_run_now
# ---------------------------------------------------------------------------
def test_first_run_defers(curator_env):
"""The FIRST observation of the curator (fresh install, no state file)
must NOT trigger an immediate run. The curator is designed to run after
a full ``interval_hours`` of skill activity, not on the first background
tick after installation. Fixes #18373.
"""
c = curator_env["curator"]
# No state file — should defer and seed last_run_at.
assert c.should_run_now() is False
state = c.load_state()
assert state.get("last_run_at") is not None, (
"first observation should seed last_run_at so the interval clock "
"starts ticking instead of firing immediately next tick"
)
# A second immediate call still returns False (seeded, not yet stale).
assert c.should_run_now() is False
def test_recent_run_blocks(curator_env):
c = curator_env["curator"]
c.save_state({
"last_run_at": datetime.now(timezone.utc).isoformat(),
"paused": False,
})
assert c.should_run_now() is False
def test_old_run_eligible(curator_env):
"""A run older than the configured interval should re-trigger. Use a
2x-interval cushion so the test doesn't become coupled to the exact
default — bumping DEFAULT_INTERVAL_HOURS shouldn't break it."""
c = curator_env["curator"]
long_ago = datetime.now(timezone.utc) - timedelta(
hours=c.get_interval_hours() * 2
)
c.save_state({"last_run_at": long_ago.isoformat(), "paused": False})
assert c.should_run_now() is True
def test_paused_blocks_even_if_stale(curator_env):
c = curator_env["curator"]
long_ago = datetime.now(timezone.utc) - timedelta(days=30)
c.save_state({"last_run_at": long_ago.isoformat(), "paused": True})
assert c.should_run_now() is False
def test_set_paused_roundtrip(curator_env):
c = curator_env["curator"]
c.set_paused(True)
assert c.is_paused() is True
c.set_paused(False)
assert c.is_paused() is False
# ---------------------------------------------------------------------------
# Automatic state transitions
# ---------------------------------------------------------------------------
def test_unused_skill_transitions_to_stale(curator_env):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "old-skill")
# Record last-use well past stale_after_days (30 default)
long_ago = (datetime.now(timezone.utc) - timedelta(days=45)).isoformat()
data = u.load_usage()
data["old-skill"] = u._empty_record()
data["old-skill"]["created_by"] = "agent"
data["old-skill"]["last_used_at"] = long_ago
data["old-skill"]["created_at"] = long_ago
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["marked_stale"] == 1
assert u.get_record("old-skill")["state"] == "stale"
def test_very_old_skill_gets_archived(curator_env):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
skill_dir = _write_skill(skills_dir, "ancient")
super_old = (datetime.now(timezone.utc) - timedelta(days=120)).isoformat()
data = u.load_usage()
data["ancient"] = u._empty_record()
data["ancient"]["created_by"] = "agent"
data["ancient"]["last_used_at"] = super_old
data["ancient"]["created_at"] = super_old
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["archived"] == 1
assert not skill_dir.exists()
assert (skills_dir / ".archive" / "ancient" / "SKILL.md").exists()
assert u.get_record("ancient")["state"] == "archived"
def test_pinned_skill_is_never_touched(curator_env):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "precious")
super_old = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat()
data = u.load_usage()
data["precious"] = u._empty_record()
data["precious"]["created_by"] = "agent"
data["precious"]["last_used_at"] = super_old
data["precious"]["created_at"] = super_old
data["precious"]["pinned"] = True
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["archived"] == 0
assert counts["marked_stale"] == 0
rec = u.get_record("precious")
assert rec["state"] == "active" # untouched
assert rec["pinned"] is True
def test_stale_skill_reactivates_on_recent_use(curator_env):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "revived")
recent = datetime.now(timezone.utc).isoformat()
data = u.load_usage()
data["revived"] = u._empty_record()
data["revived"]["created_by"] = "agent"
data["revived"]["state"] = "stale"
data["revived"]["last_used_at"] = recent
data["revived"]["created_at"] = recent
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["reactivated"] == 1
assert u.get_record("revived")["state"] == "active"
def test_new_skill_without_last_used_not_immediately_archived(curator_env):
"""A freshly-created skill with no use history should not get archived
just because last_used_at is None."""
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "fresh")
# Bump nothing — record doesn't exist yet. Curator should create it
# and fall back to created_at which is ~now.
counts = c.apply_automatic_transitions()
assert counts["archived"] == 0
assert counts["marked_stale"] == 0
assert (skills_dir / "fresh").exists()
def test_manual_skill_is_not_auto_archived(curator_env):
"""Manual skills can have usage records, but without the agent-created
marker they must stay out of curator transitions."""
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
skill_dir = _write_skill(skills_dir, "manual")
super_old = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat()
data = u.load_usage()
data["manual"] = u._empty_record()
data["manual"]["last_used_at"] = super_old
data["manual"]["created_at"] = super_old
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["checked"] == 0
assert counts["archived"] == 0
assert skill_dir.exists()
def test_bundled_skill_not_touched_by_transitions(curator_env):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "bundled")
(skills_dir / ".bundled_manifest").write_text(
"bundled:abc\n", encoding="utf-8",
)
super_old = (datetime.now(timezone.utc) - timedelta(days=500)).isoformat()
data = u.load_usage()
data["bundled"] = u._empty_record()
data["bundled"]["last_used_at"] = super_old
u.save_usage(data)
counts = c.apply_automatic_transitions()
# bundled skills are excluded from the agent-created list entirely
assert counts["checked"] == 0
assert (skills_dir / "bundled").exists() # never moved
# ---------------------------------------------------------------------------
# run_curator_review orchestration
# ---------------------------------------------------------------------------
def test_run_review_records_state(curator_env):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
u.mark_agent_created("a")
result = c.run_curator_review(synchronous=True)
assert "started_at" in result
state = c.load_state()
assert state["last_run_at"] is not None
assert state["run_count"] >= 1
assert state["last_run_summary"] is not None
def test_dry_run_does_not_advance_state(curator_env, monkeypatch):
"""Dry-run previews must not bump last_run_at or run_count. A preview
shouldn't defer the next scheduled real pass or look like a real run in
`hermes curator status`. Fixes #18373.
"""
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
u.mark_agent_created("a")
# Stub the LLM so the test doesn't need a provider.
monkeypatch.setattr(
c, "_run_llm_review",
lambda prompt: {
"final": "", "summary": "dry preview", "model": "", "provider": "",
"tool_calls": [], "error": None,
},
)
c.run_curator_review(synchronous=True, dry_run=True)
state = c.load_state()
assert state.get("last_run_at") is None, "dry-run must not seed last_run_at"
assert state.get("run_count", 0) == 0, "dry-run must not bump run_count"
assert "dry-run" in (state.get("last_run_summary") or ""), (
"dry-run summary should be labeled so status output is unambiguous"
)
def test_dry_run_injects_report_only_banner(curator_env, monkeypatch):
"""The dry-run prompt must carry a banner instructing the LLM not to
call any mutating tool. This is defense in depth — the caller also
skips automatic transitions — but the LLM prompt is the only guard
against the model calling skill_manage directly."""
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
u.mark_agent_created("a")
captured = {}
def _stub(prompt):
captured["prompt"] = prompt
return {"final": "", "summary": "s", "model": "", "provider": "",
"tool_calls": [], "error": None}
monkeypatch.setattr(c, "_run_llm_review", _stub)
c.run_curator_review(synchronous=True, dry_run=True)
assert "DRY-RUN" in captured["prompt"]
assert "DO NOT" in captured["prompt"]
def test_dry_run_skips_automatic_transitions(curator_env, monkeypatch):
"""Dry-run must not call apply_automatic_transitions — the auto pass
archives skills deterministically, and a preview must not touch the
filesystem."""
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
u.mark_agent_created("a")
called = {"n": 0}
def _explode(*_a, **_kw):
called["n"] += 1
return {"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0}
monkeypatch.setattr(c, "apply_automatic_transitions", _explode)
monkeypatch.setattr(
c, "_run_llm_review",
lambda p: {"final": "", "summary": "s", "model": "", "provider": "",
"tool_calls": [], "error": None},
)
c.run_curator_review(synchronous=True, dry_run=True)
assert called["n"] == 0, "dry-run must skip apply_automatic_transitions"
def test_run_review_synchronous_invokes_llm_stub(curator_env, monkeypatch):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
u.mark_agent_created("a")
calls = []
def _stub(prompt):
calls.append(prompt)
return {
"final": "stubbed-summary",
"summary": "stubbed-summary",
"model": "stub-model",
"provider": "stub-provider",
"tool_calls": [],
"error": None,
}
monkeypatch.setattr(c, "_run_llm_review", _stub)
captured = []
c.run_curator_review(on_summary=lambda s: captured.append(s), synchronous=True)
assert len(calls) == 1
assert "skill CURATOR" in calls[0] or "CURATOR" in calls[0]
assert captured # on_summary was called
assert any("stubbed-summary" in s for s in captured)
def test_run_review_skips_llm_when_no_candidates(curator_env, monkeypatch):
c = curator_env["curator"]
# No skills in the dir → no candidates
calls = []
monkeypatch.setattr(
c, "_run_llm_review",
lambda prompt: (calls.append(prompt), "never-called")[1],
)
captured = []
c.run_curator_review(on_summary=lambda s: captured.append(s), synchronous=True)
assert calls == [] # LLM not invoked
assert any("skipped" in s for s in captured)
def test_maybe_run_curator_respects_disabled(curator_env, monkeypatch):
c = curator_env["curator"]
monkeypatch.setattr(c, "_load_config", lambda: {"enabled": False})
result = c.maybe_run_curator()
assert result is None
def test_maybe_run_curator_enforces_idle_gate(curator_env, monkeypatch):
c = curator_env["curator"]
monkeypatch.setattr(c, "_load_config", lambda: {"min_idle_hours": 2})
# idle less than the threshold
result = c.maybe_run_curator(idle_for_seconds=60.0)
assert result is None
def test_maybe_run_curator_runs_when_eligible(curator_env, monkeypatch):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
u.mark_agent_created("a")
# Seed last_run_at far in the past so the interval gate opens — the
# "no state" path intentionally defers the first run now (#18373).
long_ago = datetime.now(timezone.utc) - timedelta(hours=c.get_interval_hours() * 2)
c.save_state({"last_run_at": long_ago.isoformat(), "paused": False})
# Force idle over threshold
result = c.maybe_run_curator(idle_for_seconds=99999.0)
assert result is not None
assert "started_at" in result
def test_maybe_run_curator_defers_on_fresh_install(curator_env):
"""Fresh install (no curator state file) must NOT fire the curator on
the first gateway tick. The first observation seeds last_run_at and
returns None. Fixes #18373."""
c = curator_env["curator"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "a")
# Infinite idle — the only thing that should block the run is the new
# deferred-first-run gate.
result = c.maybe_run_curator(idle_for_seconds=99999.0)
assert result is None
# And the next tick still defers (we seeded last_run_at to "now").
result2 = c.maybe_run_curator(idle_for_seconds=99999.0)
assert result2 is None
def test_maybe_run_curator_swallows_exceptions(curator_env, monkeypatch):
c = curator_env["curator"]
def explode():
raise RuntimeError("boom")
monkeypatch.setattr(c, "should_run_now", explode)
# Must not raise
assert c.maybe_run_curator() is None
# ---------------------------------------------------------------------------
# Persistence
# ---------------------------------------------------------------------------
def test_state_file_survives_corrupt_read(curator_env):
c = curator_env["curator"]
c._state_file().write_text("not json", encoding="utf-8")
# Must fall back to default, not raise
assert c.load_state() == c._default_state()
def test_state_atomic_write_no_tmp_leftovers(curator_env):
c = curator_env["curator"]
c.save_state({"paused": True})
parent = c._state_file().parent
for p in parent.iterdir():
assert not p.name.startswith(".curator_state_"), f"tmp leftover: {p.name}"
def test_state_preserves_last_report_path(curator_env):
c = curator_env["curator"]
c.save_state({
"last_run_at": "2026-04-30T12:00:00+00:00",
"last_run_summary": "ok",
"last_report_path": "/tmp/curator-report",
"paused": False,
"run_count": 1,
})
state = c.load_state()
assert state["last_report_path"] == "/tmp/curator-report"
def test_curator_review_prompt_has_invariants():
"""Core invariants must be in the review prompt text."""
from agent.curator import CURATOR_REVIEW_PROMPT
assert "MUST NOT" in CURATOR_REVIEW_PROMPT or "DO NOT" in CURATOR_REVIEW_PROMPT
assert "bundled" in CURATOR_REVIEW_PROMPT.lower()
assert "delete" in CURATOR_REVIEW_PROMPT.lower()
assert "pinned" in CURATOR_REVIEW_PROMPT.lower()
# Must describe the actions the reviewer can take. The exact vocabulary
# has tightened over time (the umbrella-first prompt drops 'keep' as a
# first-class decision verb, since passive keep-everything is the
# failure mode the prompt is trying to avoid), but the core merge /
# archive / patch trio must remain callable.
for verb in ("patch", "archive"):
assert verb in CURATOR_REVIEW_PROMPT.lower()
# Must mention consolidation (possibly via "merge" or "consolidat")
assert "consolidat" in CURATOR_REVIEW_PROMPT.lower() or "merge" in CURATOR_REVIEW_PROMPT.lower()
def test_curator_review_prompt_points_at_existing_tools_only():
"""The review prompt must rely on existing tools (skill_manage + terminal)
and must NOT reference bespoke curator tools that are not registered
model tools."""
from agent.curator import CURATOR_REVIEW_PROMPT
assert "skill_manage" in CURATOR_REVIEW_PROMPT
assert "skills_list" in CURATOR_REVIEW_PROMPT
assert "skill_view" in CURATOR_REVIEW_PROMPT
assert "terminal" in CURATOR_REVIEW_PROMPT.lower()
# These would be nice but aren't actually registered as tools — the
# curator uses skill_manage + terminal mv instead.
assert "archive_skill" not in CURATOR_REVIEW_PROMPT
assert "pin_skill" not in CURATOR_REVIEW_PROMPT
def test_curator_does_not_instruct_model_to_pin():
"""Pinning is a user opt-out, not a model decision. The prompt should
not tell the reviewer to pin skills autonomously."""
from agent.curator import CURATOR_REVIEW_PROMPT
# "pinned" appears in the invariant ("skip pinned skills"), but "pin"
# as a decision verb should not.
lines = CURATOR_REVIEW_PROMPT.split("\n")
decision_block = "\n".join(
l for l in lines
if l.strip().startswith(("keep", "patch", "archive", "consolidate", "pin "))
)
# No standalone "pin" action line
assert not any(l.strip().startswith("pin ") for l in lines), (
f"Found a pin action line in:\n{decision_block}"
)
def test_curator_review_prompt_is_umbrella_first():
"""The curator prompt must push umbrella-building / class-level thinking,
not pair-level 'are these two the same?' analysis."""
from agent.curator import CURATOR_REVIEW_PROMPT
lower = CURATOR_REVIEW_PROMPT.lower()
# Must frame the task as active umbrella-building, not a passive audit.
assert "umbrella" in lower, (
"must use UMBRELLA framing — the class-first abstraction the curator "
"is designed to produce"
)
# Must tell the reviewer not to stop at pair-level distinctness.
assert "class" in lower, "must reference class-level thinking"
# Must cover the three consolidation methods explicitly
assert "references/" in CURATOR_REVIEW_PROMPT, (
"must name references/ as a demotion target for session-specific content"
)
# templates/ and scripts/ make the umbrella a real class-level skill
assert "templates/" in CURATOR_REVIEW_PROMPT
assert "scripts/" in CURATOR_REVIEW_PROMPT
# Must say the counter argument: usage=0 is not a reason to skip
assert "use_count" in CURATOR_REVIEW_PROMPT or "counter" in lower, (
"must pre-empt the 'usage counters are zero, I can't judge' bailout"
)
def test_curator_review_prompt_preserves_skill_package_integrity():
"""Consolidation must not flatten package skills and break linked files."""
from agent.curator import CURATOR_REVIEW_PROMPT
lower = CURATOR_REVIEW_PROMPT.lower()
assert "complete" in lower and "directory package" in lower
assert "not a new skill root" in lower
assert "do not flatten only skill.md" in lower
assert "rewrite" in lower and "new paths" in lower
assert "archive the entire original skill package unchanged" in lower
for dirname in ("references/", "templates/", "scripts/", "assets/"):
assert dirname in CURATOR_REVIEW_PROMPT
def test_curator_review_prompt_offers_support_file_actions():
"""Support-file demotion (references/templates/scripts) must be one of
the three consolidation methods, alongside merge-into-existing and
create-new-umbrella."""
from agent.curator import CURATOR_REVIEW_PROMPT
# skill_manage action=write_file is how references/ are added to an
# existing skill — this is the create-adjacent action the curator needs
# to demote narrow siblings without touching their SKILL.md.
assert "write_file" in CURATOR_REVIEW_PROMPT
# Must offer creating a brand-new umbrella when no existing one fits
assert "action=create" in CURATOR_REVIEW_PROMPT or "create a new umbrella" in CURATOR_REVIEW_PROMPT.lower()
def test_cli_unpin_refuses_bundled_skill(curator_env, capsys):
"""hermes curator unpin must refuse bundled/hub skills too (matches pin)."""
from hermes_cli import curator as cli
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "ship-skill")
(skills_dir / ".bundled_manifest").write_text(
"ship-skill:abc\n", encoding="utf-8",
)
class _A:
skill = "ship-skill"
rc = cli._cmd_unpin(_A())
captured = capsys.readouterr()
assert rc == 1
assert "bundled" in captured.out.lower() or "hub" in captured.out.lower()
def test_cli_pin_refuses_bundled_skill(curator_env, capsys):
from hermes_cli import curator as cli
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "ship-skill")
(skills_dir / ".bundled_manifest").write_text(
"ship-skill:abc\n", encoding="utf-8",
)
class _A:
skill = "ship-skill"
rc = cli._cmd_pin(_A())
captured = capsys.readouterr()
assert rc == 1
assert "bundled" in captured.out.lower() or "hub" in captured.out.lower()
# ---------------------------------------------------------------------------
# curator review-model resolution (canonical auxiliary.curator slot)
#
# Curator was unified with the rest of the aux task system in Apr 2026 so
# `hermes model` → auxiliary picker, the dashboard Models tab, and the full
# per-task config (timeout, base_url, api_key, extra_body) all work for it.
# Voscko report: curator.auxiliary.{provider,model} was advertised but never
# read. Fix wires curator through auxiliary.curator with a legacy fallback.
# ---------------------------------------------------------------------------
def test_review_model_defaults_to_main_when_slot_is_auto(curator_env):
"""auxiliary.curator absent (or auto/empty) → use main model.provider/model."""
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
}
assert curator._resolve_review_model(cfg) == ("openrouter", "openai/gpt-5.5")
# Explicit auto/empty slot — still main model.
cfg["auxiliary"] = {"curator": {"provider": "auto", "model": ""}}
assert curator._resolve_review_model(cfg) == ("openrouter", "openai/gpt-5.5")
def test_review_model_honors_auxiliary_curator_slot(curator_env):
"""auxiliary.curator.{provider,model} fully set → that pair wins."""
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"auxiliary": {
"curator": {
"provider": "openrouter",
"model": "openai/gpt-5.4-mini",
},
},
}
assert curator._resolve_review_model(cfg) == (
"openrouter", "openai/gpt-5.4-mini",
)
def test_review_runtime_passes_auxiliary_curator_credentials(curator_env):
"""Per-slot api_key/base_url must ride into resolve_runtime_provider (not main-only creds)."""
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"auxiliary": {
"curator": {
"provider": "custom",
"model": "local-mini",
"api_key": "sk-curator-only",
"base_url": "http://localhost:11434/v1",
},
},
}
binding = curator._resolve_review_runtime(cfg)
assert binding.provider == "custom"
assert binding.model == "local-mini"
assert binding.explicit_api_key == "sk-curator-only"
assert binding.explicit_base_url == "http://localhost:11434/v1"
def test_review_runtime_strips_blank_aux_credentials(curator_env):
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"auxiliary": {
"curator": {
"provider": "openrouter",
"model": "x/y",
"api_key": " ",
"base_url": "",
},
},
}
binding = curator._resolve_review_runtime(cfg)
assert binding.explicit_api_key is None
assert binding.explicit_base_url is None
def test_review_runtime_ignores_auxiliary_credentials_when_using_main(curator_env):
"""Falling through to main model must not pick up stray auxiliary.curator secrets."""
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"auxiliary": {
"curator": {
"provider": "auto",
"model": "",
"api_key": "must-not-leak",
"base_url": "http://curator-slot-ignored/",
},
},
}
binding = curator._resolve_review_runtime(cfg)
assert (binding.provider, binding.model) == ("openrouter", "openai/gpt-5.5")
assert binding.explicit_api_key is None
assert binding.explicit_base_url is None
def test_review_runtime_legacy_auxiliary_carry_credentials(curator_env, caplog):
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"curator": {
"auxiliary": {
"provider": "custom",
"model": "m",
"api_key": "legacy-key",
"base_url": "http://legacy/v1",
},
},
}
import logging
with caplog.at_level(logging.INFO, logger="agent.curator"):
binding = curator._resolve_review_runtime(cfg)
assert binding.explicit_api_key == "legacy-key"
assert binding.explicit_base_url == "http://legacy/v1"
assert any("deprecated curator.auxiliary" in rec.message for rec in caplog.records)
def test_review_model_auxiliary_curator_partial_override_falls_back(curator_env):
"""Only one of slot provider/model set → fall back to the main pair.
Prevents half-configured overrides from sending an empty side to
resolve_runtime_provider.
"""
curator = curator_env["curator"]
base_main = {"provider": "openrouter", "default": "openai/gpt-5.5"}
cfg_provider_only = {
"model": dict(base_main),
"auxiliary": {"curator": {"provider": "openrouter", "model": ""}},
}
assert curator._resolve_review_model(cfg_provider_only) == (
"openrouter", "openai/gpt-5.5",
)
cfg_model_only = {
"model": dict(base_main),
"auxiliary": {"curator": {"provider": "auto", "model": "gpt-5.4-mini"}},
}
assert curator._resolve_review_model(cfg_model_only) == (
"openrouter", "openai/gpt-5.5",
)
def test_review_model_legacy_curator_auxiliary_still_works(curator_env, caplog):
"""Pre-unification users set curator.auxiliary.{provider,model} — honor it.
Emits a deprecation log line but keeps their config working.
"""
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"curator": {
"auxiliary": {
"provider": "openrouter",
"model": "openai/gpt-5.4-mini",
},
},
}
import logging
with caplog.at_level(logging.INFO, logger="agent.curator"):
result = curator._resolve_review_model(cfg)
assert result == ("openrouter", "openai/gpt-5.4-mini")
assert any(
"deprecated curator.auxiliary" in rec.message for rec in caplog.records
), "expected deprecation warning when legacy curator.auxiliary is used"
def test_review_model_new_slot_wins_over_legacy(curator_env):
"""When BOTH new and legacy are set, the canonical slot wins."""
curator = curator_env["curator"]
cfg = {
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
"auxiliary": {
"curator": {"provider": "nous", "model": "new-winner"},
},
"curator": {
"auxiliary": {"provider": "openrouter", "model": "legacy-loser"},
},
}
assert curator._resolve_review_model(cfg) == ("nous", "new-winner")
def test_review_model_handles_missing_sections(curator_env):
"""Missing auxiliary/curator sections never raise — fall back cleanly."""
curator = curator_env["curator"]
cfg = {"model": {"provider": "anthropic", "model": "claude-sonnet-4-6"}}
assert curator._resolve_review_model(cfg) == (
"anthropic", "claude-sonnet-4-6",
)
# Completely empty config → ("auto", "") — resolve_runtime_provider
# handles the auto-detection chain from there.
assert curator._resolve_review_model({}) == ("auto", "")
def test_curator_slot_is_canonical_aux_task():
"""Curator must be a first-class slot in every aux-task registry.
Four sources of truth, all checked by the shared registry test
(test_aux_config.py) for the main tasks — this test pins `curator`
specifically so the unification doesn't silently regress.
"""
from hermes_cli.config import DEFAULT_CONFIG
from hermes_cli.main import _AUX_TASKS
from hermes_cli.web_server import _AUX_TASK_SLOTS
# 1. DEFAULT_CONFIG.auxiliary — schema source
assert "curator" in DEFAULT_CONFIG["auxiliary"], \
"curator missing from DEFAULT_CONFIG['auxiliary']"
slot = DEFAULT_CONFIG["auxiliary"]["curator"]
assert slot["provider"] == "auto"
assert slot["model"] == ""
assert slot["timeout"] > 0, "curator timeout should be set (reviews run long)"
# 2. hermes_cli/main.py _AUX_TASKS — CLI picker
aux_keys = {k for k, _name, _desc in _AUX_TASKS}
assert "curator" in aux_keys, "curator missing from _AUX_TASKS (CLI picker)"
# 3. hermes_cli/web_server.py _AUX_TASK_SLOTS — REST API allowlist
assert "curator" in _AUX_TASK_SLOTS, \
"curator missing from _AUX_TASK_SLOTS (dashboard REST API)"
# 4. web/src/pages/ModelsPage.tsx is checked at build time; the tsx
# array and this tuple share a ``Must match _AUX_TASK_SLOTS`` comment.
+56
View File
@@ -0,0 +1,56 @@
"""Regression tests for curator skill activity timestamps."""
import importlib
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
def _write_skill(skills_dir: Path, name: str) -> None:
skill_dir = skills_dir / name
skill_dir.mkdir(parents=True, exist_ok=True)
(skill_dir / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: test skill\n---\n\n# {name}\n",
encoding="utf-8",
)
@pytest.fixture
def curator_modules(tmp_path, monkeypatch):
home = tmp_path / ".hermes"
(home / "skills").mkdir(parents=True)
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
import tools.skill_usage as skill_usage
import agent.curator as curator
importlib.reload(skill_usage)
importlib.reload(curator)
return home, skill_usage, curator
def test_recent_view_activity_prevents_false_stale_transition(curator_modules, monkeypatch):
home, skill_usage, curator = curator_modules
skills_dir = home / "skills"
_write_skill(skills_dir, "recently-viewed")
now = datetime(2026, 4, 30, tzinfo=timezone.utc)
created_at = (now - timedelta(days=60)).isoformat()
last_viewed_at = (now - timedelta(days=1)).isoformat()
skill_usage.save_usage({
"recently-viewed": {
"created_at": created_at,
"last_viewed_at": last_viewed_at,
"view_count": 1,
"state": "active",
}
})
monkeypatch.setattr(curator, "get_stale_after_days", lambda: 30)
monkeypatch.setattr(curator, "get_archive_after_days", lambda: 90)
counts = curator.apply_automatic_transitions(now=now)
assert counts["marked_stale"] == 0
assert skill_usage.get_record("recently-viewed")["state"] == "active"
+594
View File
@@ -0,0 +1,594 @@
"""Tests for agent/curator_backup.py — snapshot + rollback of the skills tree."""
from __future__ import annotations
import importlib
import json
import os
import sys
import tarfile
import tempfile
from pathlib import Path
import pytest
@pytest.fixture
def backup_env(monkeypatch, tmp_path):
"""Isolate HERMES_HOME + reload modules so every test starts clean."""
home = tmp_path / ".hermes"
home.mkdir()
(home / "skills").mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
# Reload so get_hermes_home picks up the env var fresh.
import hermes_constants
importlib.reload(hermes_constants)
from agent import curator_backup
importlib.reload(curator_backup)
return {"home": home, "skills": home / "skills", "cb": curator_backup}
def _write_skill(skills_dir: Path, name: str, body: str = "body") -> Path:
d = skills_dir / name
d.mkdir(parents=True, exist_ok=True)
(d / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: t\nversion: 1.0\n---\n\n{body}\n",
encoding="utf-8",
)
return d
# ---------------------------------------------------------------------------
# snapshot_skills
# ---------------------------------------------------------------------------
def test_snapshot_creates_tarball_and_manifest(backup_env):
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
_write_skill(backup_env["skills"], "beta")
snap = cb.snapshot_skills(reason="test")
assert snap is not None, "snapshot should succeed with a populated skills dir"
assert (snap / "skills.tar.gz").exists()
manifest = json.loads((snap / "manifest.json").read_text())
assert manifest["reason"] == "test"
assert manifest["skill_files"] == 2
assert manifest["archive_bytes"] > 0
def test_snapshot_excludes_backups_dir_itself(backup_env):
"""The backup must NOT contain .curator_backups/ — that would recurse
with every subsequent snapshot and balloon disk usage."""
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
snap1 = cb.snapshot_skills(reason="first")
assert snap1 is not None
snap2 = cb.snapshot_skills(reason="second")
assert snap2 is not None
with tarfile.open(snap2 / "skills.tar.gz") as tf:
names = tf.getnames()
assert not any(n.startswith(".curator_backups") for n in names), (
"second snapshot must not contain the first snapshot recursively"
)
def test_snapshot_excludes_hub_dir(backup_env):
""".hub/ is managed by the skills hub. Rolling it back would break
lockfile invariants, so the snapshot omits it entirely."""
cb = backup_env["cb"]
hub = backup_env["skills"] / ".hub"
hub.mkdir()
(hub / "lock.json").write_text("{}")
_write_skill(backup_env["skills"], "alpha")
snap = cb.snapshot_skills(reason="t")
assert snap is not None
with tarfile.open(snap / "skills.tar.gz") as tf:
names = tf.getnames()
assert not any(n.startswith(".hub") for n in names)
def test_snapshot_disabled_returns_none(backup_env, monkeypatch):
cb = backup_env["cb"]
monkeypatch.setattr(cb, "is_enabled", lambda: False)
_write_skill(backup_env["skills"], "alpha")
assert cb.snapshot_skills() is None
# And no backup dir should have been created
assert not (backup_env["skills"] / ".curator_backups").exists()
def test_snapshot_uniquifies_when_same_second(backup_env, monkeypatch):
"""Two snapshots in the same wallclock second must not clobber each
other. The module appends a counter to the second snapshot's id."""
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
frozen = "2026-05-01T12-00-00Z"
monkeypatch.setattr(cb, "_utc_id", lambda now=None: frozen)
s1 = cb.snapshot_skills(reason="a")
s2 = cb.snapshot_skills(reason="b")
assert s1 is not None and s2 is not None
assert s1.name == frozen
assert s2.name == f"{frozen}-01"
def test_snapshot_prunes_to_keep_count(backup_env, monkeypatch):
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
monkeypatch.setattr(cb, "get_keep", lambda: 3)
# Create 5 snapshots with monotonically increasing fake ids
ids = [f"2026-05-0{i}T00-00-00Z" for i in range(1, 6)]
for i, fid in enumerate(ids):
monkeypatch.setattr(cb, "_utc_id", lambda now=None, _f=fid: _f)
cb.snapshot_skills(reason=f"n{i}")
remaining = sorted(p.name for p in (backup_env["skills"] / ".curator_backups").iterdir())
# Newest 3 kept (lex order == date order for this id format)
assert remaining == ids[2:], f"expected newest 3, got {remaining}"
# ---------------------------------------------------------------------------
# list_backups / _resolve_backup
# ---------------------------------------------------------------------------
def test_list_backups_empty(backup_env):
cb = backup_env["cb"]
assert cb.list_backups() == []
def test_list_backups_returns_manifest_data(backup_env):
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
cb.snapshot_skills(reason="m1")
rows = cb.list_backups()
assert len(rows) == 1
assert rows[0]["reason"] == "m1"
assert rows[0]["skill_files"] == 1
def test_resolve_backup_newest_when_no_id(backup_env, monkeypatch):
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
ids = ["2026-05-01T00-00-00Z", "2026-05-02T00-00-00Z"]
for fid in ids:
monkeypatch.setattr(cb, "_utc_id", lambda now=None, _f=fid: _f)
cb.snapshot_skills()
resolved = cb._resolve_backup(None)
assert resolved is not None
assert resolved.name == "2026-05-02T00-00-00Z", (
"resolve(None) must return newest regular snapshot"
)
def test_resolve_backup_unknown_id_returns_none(backup_env):
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
cb.snapshot_skills()
assert cb._resolve_backup("not-an-id") is None
# ---------------------------------------------------------------------------
# rollback
# ---------------------------------------------------------------------------
def test_rollback_restores_deleted_skill(backup_env):
"""The whole point of this feature: user loses a skill, rollback
brings it back."""
cb = backup_env["cb"]
skills = backup_env["skills"]
user_skill = _write_skill(skills, "my-personal-workflow", body="important content")
cb.snapshot_skills(reason="pre-simulated-curator")
# Simulate curator archiving it out of existence
import shutil as _sh
_sh.rmtree(user_skill)
assert not user_skill.exists()
ok, msg, _ = cb.rollback()
assert ok, f"rollback failed: {msg}"
assert user_skill.exists(), "my-personal-workflow should be restored"
assert "important content" in (user_skill / "SKILL.md").read_text()
def test_rollback_is_itself_undoable(backup_env):
"""A rollback creates its own safety snapshot before replacing the
tree, so the user can undo a mistaken rollback. The safety snapshot
is a real tarball with reason='pre-rollback to <id>' — it's
listed by list_backups() just like any other snapshot and can be
restored the same way."""
cb = backup_env["cb"]
skills = backup_env["skills"]
_write_skill(skills, "v1")
cb.snapshot_skills(reason="snapshot-of-v1")
# Overwrite with a new skill state
import shutil as _sh
_sh.rmtree(skills / "v1")
_write_skill(skills, "v2")
ok, _, _ = cb.rollback()
assert ok
assert (skills / "v1").exists()
# list_backups should show a safety snapshot tagged "pre-rollback to <target-id>"
rows = cb.list_backups()
pre_rollback_entries = [r for r in rows if "pre-rollback" in (r.get("reason") or "")]
assert len(pre_rollback_entries) >= 1, (
f"expected a pre-rollback safety snapshot in list_backups(), got: "
f"{[(r.get('id'), r.get('reason')) for r in rows]}"
)
# And the transient staging dir must be gone (it's implementation detail)
backups_dir = skills / ".curator_backups"
staging_dirs = [p for p in backups_dir.iterdir() if p.name.startswith(".rollback-staging-")]
assert staging_dirs == [], (
f"staging dir should be cleaned up on success, got: {staging_dirs}"
)
def test_rollback_no_snapshots_returns_error(backup_env):
cb = backup_env["cb"]
ok, msg, _ = cb.rollback()
assert not ok
assert "no matching backup" in msg.lower() or "no snapshot" in msg.lower()
def test_rollback_rejects_unsafe_tarball(backup_env, monkeypatch):
"""Tarballs with absolute paths or .. components must be refused even
if someone crafts a malicious snapshot. Defense in depth — normal
curator snapshots never produce these."""
cb = backup_env["cb"]
skills = backup_env["skills"]
_write_skill(skills, "alpha")
cb.snapshot_skills(reason="legit")
# Hand-craft a malicious tarball replacing the legit one
rows = cb.list_backups()
snap_dir = Path(rows[0]["path"])
mal = snap_dir / "skills.tar.gz"
mal.unlink()
with tarfile.open(mal, "w:gz") as tf:
evil = tempfile.NamedTemporaryFile(delete=False, suffix=".md")
evil.write(b"evil")
evil.close()
tf.add(evil.name, arcname="../../etc/evil.md")
os.unlink(evil.name)
ok, msg, _ = cb.rollback()
assert not ok
assert "unsafe" in msg.lower() or "refus" in msg.lower() or "extract" in msg.lower()
# ---------------------------------------------------------------------------
# Integration with run_curator_review
# ---------------------------------------------------------------------------
def test_real_run_takes_pre_snapshot(backup_env, monkeypatch):
"""A real (non-dry) curator pass must snapshot the tree before calling
apply_automatic_transitions. This is the safety net #18373 asked for."""
cb = backup_env["cb"]
skills = backup_env["skills"]
_write_skill(skills, "alpha")
# Reload curator module against the freshly-env'd hermes_constants
from agent import curator
importlib.reload(curator)
# Stub out LLM review and auto transitions — we only care about the
# snapshot side-effect.
monkeypatch.setattr(
curator, "_run_llm_review",
lambda p: {"final": "", "summary": "s", "model": "", "provider": "",
"tool_calls": [], "error": None},
)
monkeypatch.setattr(
curator, "apply_automatic_transitions",
lambda now=None: {"checked": 1, "marked_stale": 0, "archived": 0, "reactivated": 0},
)
curator.run_curator_review(synchronous=True)
# Pre-run snapshot should exist
rows = cb.list_backups()
assert any(r.get("reason") == "pre-curator-run" for r in rows), (
f"expected a pre-curator-run snapshot, got {[r.get('reason') for r in rows]}"
)
def test_dry_run_skips_snapshot(backup_env, monkeypatch):
"""Dry-run previews must not spend disk on a snapshot — they don't
mutate anything, so there's nothing to back up."""
cb = backup_env["cb"]
skills = backup_env["skills"]
_write_skill(skills, "alpha")
from agent import curator
importlib.reload(curator)
monkeypatch.setattr(
curator, "_run_llm_review",
lambda p: {"final": "", "summary": "s", "model": "", "provider": "",
"tool_calls": [], "error": None},
)
curator.run_curator_review(synchronous=True, dry_run=True)
rows = cb.list_backups()
assert not any(r.get("reason") == "pre-curator-run" for r in rows), (
"dry-run must not create a pre-run snapshot"
)
# ---------------------------------------------------------------------------
# cron-jobs backup + rollback (the part issue #18671's follow-up adds)
# ---------------------------------------------------------------------------
def _write_cron_jobs(home: Path, jobs: list) -> Path:
"""Write a synthetic cron/jobs.json under HERMES_HOME. Returns the path.
Mirrors cron.jobs.save_jobs() wrapper shape: `{"jobs": [...], "updated_at": ...}`.
"""
cron_dir = home / "cron"
cron_dir.mkdir(parents=True, exist_ok=True)
path = cron_dir / "jobs.json"
path.write_text(
json.dumps({"jobs": jobs, "updated_at": "2026-05-01T00:00:00Z"}, indent=2),
encoding="utf-8",
)
return path
def _reload_cron_jobs(home: Path):
"""Reload cron.jobs so its module-level HERMES_DIR picks up the tmp HOME."""
import hermes_constants
importlib.reload(hermes_constants)
if "cron.jobs" in sys.modules:
import cron.jobs as _cj
importlib.reload(_cj)
else:
import cron.jobs as _cj # noqa: F401
import cron.jobs as cj
return cj
def test_snapshot_includes_cron_jobs(backup_env):
"""With a cron/jobs.json present, snapshot writes cron-jobs.json and records it in manifest."""
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
_write_cron_jobs(backup_env["home"], [
{"id": "job-a", "name": "a", "schedule": "every 1h", "skills": ["alpha"]},
{"id": "job-b", "name": "b", "schedule": "every 2h", "skill": "alpha"},
])
snap = cb.snapshot_skills(reason="test")
assert snap is not None
assert (snap / cb.CRON_JOBS_FILENAME).exists()
mf = json.loads((snap / "manifest.json").read_text(encoding="utf-8"))
assert mf["cron_jobs"]["backed_up"] is True
assert mf["cron_jobs"]["jobs_count"] == 2
def test_snapshot_without_cron_jobs_file_still_succeeds(backup_env):
"""No cron/jobs.json on disk → snapshot succeeds, manifest records absence."""
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
# Deliberately do not create ~/.hermes/cron/jobs.json
snap = cb.snapshot_skills(reason="test")
assert snap is not None
assert not (snap / cb.CRON_JOBS_FILENAME).exists()
mf = json.loads((snap / "manifest.json").read_text(encoding="utf-8"))
assert mf["cron_jobs"]["backed_up"] is False
assert "cron/jobs.json" in mf["cron_jobs"]["reason"]
def test_snapshot_cron_jobs_malformed_json_still_captured(backup_env):
"""Malformed jobs.json is still copied to the snapshot (fidelity over
validation); the manifest notes the parse warning."""
cb = backup_env["cb"]
_write_skill(backup_env["skills"], "alpha")
(backup_env["home"] / "cron").mkdir()
(backup_env["home"] / "cron" / "jobs.json").write_text("{oh no", encoding="utf-8")
snap = cb.snapshot_skills(reason="test")
assert snap is not None
# Raw file was copied even though we couldn't parse it
assert (snap / cb.CRON_JOBS_FILENAME).read_text() == "{oh no"
mf = json.loads((snap / "manifest.json").read_text(encoding="utf-8"))
assert mf["cron_jobs"]["backed_up"] is True
assert mf["cron_jobs"]["jobs_count"] == 0
assert "parse_warning" in mf["cron_jobs"]
def test_rollback_restores_cron_skill_links(backup_env):
"""End-to-end: snapshot with job [alpha,beta], curator-style in-place
rewrite to [umbrella], then rollback → skills restored to [alpha,beta]."""
cb = backup_env["cb"]
home = backup_env["home"]
_write_skill(backup_env["skills"], "alpha")
_write_skill(backup_env["skills"], "beta")
_write_skill(backup_env["skills"], "umbrella")
cj = _reload_cron_jobs(home)
cj.create_job(name="weekly", prompt="p", schedule="every 7d",
skills=["alpha", "beta"])
snap = cb.snapshot_skills(reason="pre-curator-run")
assert snap is not None
# Simulate the curator's in-place cron rewrite after consolidation
cj.rewrite_skill_refs(
consolidated={"alpha": "umbrella", "beta": "umbrella"},
pruned=[],
)
live_after_curator = cj.load_jobs()
assert live_after_curator[0]["skills"] == ["umbrella"]
# Now roll back
ok, msg, _ = cb.rollback(backup_id=snap.name)
assert ok, msg
assert "cron links" in msg
live_after_rollback = cj.load_jobs()
# skills restored; legacy `skill` mirror follows first element
assert live_after_rollback[0]["skills"] == ["alpha", "beta"]
def test_rollback_only_touches_skill_fields(backup_env):
"""Every field other than skills/skill must remain untouched across rollback.
Schedule, enabled, prompt, timestamps — all live state, hands off."""
cb = backup_env["cb"]
home = backup_env["home"]
_write_skill(backup_env["skills"], "alpha")
# Hand-rolled jobs.json with varied fields (no real create_job — we want
# exact field control).
_write_cron_jobs(home, [{
"id": "stable-id",
"name": "original-name",
"prompt": "original prompt",
"schedule": "every 1h",
"skills": ["alpha"],
"enabled": True,
"last_run_at": "2026-04-01T00:00:00Z",
}])
snap = cb.snapshot_skills(reason="pre-curator-run")
assert snap is not None
# User/scheduler activity AFTER the snapshot: rename the job, change
# the schedule, update timestamps, and (curator) rewrite the skills list.
cj = _reload_cron_jobs(home)
jobs = cj.load_jobs()
jobs[0]["name"] = "renamed-since-snapshot"
jobs[0]["schedule"] = "every 30m"
jobs[0]["last_run_at"] = "2026-05-01T12:00:00Z"
jobs[0]["skills"] = ["umbrella"] # pretend curator did this
cj.save_jobs(jobs)
ok, _, _ = cb.rollback(backup_id=snap.name)
assert ok
after = cj.load_jobs()
job = after[0]
# skills: restored
assert job["skills"] == ["alpha"]
# everything else: untouched (live state preserved)
assert job["name"] == "renamed-since-snapshot"
assert job["schedule"] == "every 30m"
assert job["last_run_at"] == "2026-05-01T12:00:00Z"
assert job["prompt"] == "original prompt"
def test_rollback_skips_jobs_the_user_deleted(backup_env):
"""If the user deleted a cron job after the snapshot, rollback must
NOT resurrect it — the user's delete is a later, explicit choice."""
cb = backup_env["cb"]
home = backup_env["home"]
_write_skill(backup_env["skills"], "alpha")
_write_cron_jobs(home, [
{"id": "keep-me", "name": "keep", "schedule": "every 1h", "skills": ["alpha"]},
{"id": "delete-me", "name": "gone", "schedule": "every 1h", "skills": ["alpha"]},
])
snap = cb.snapshot_skills(reason="pre-curator-run")
# User deletes one job after the snapshot
cj = _reload_cron_jobs(home)
cj.save_jobs([j for j in cj.load_jobs() if j["id"] != "delete-me"])
ok, _, _ = cb.rollback(backup_id=snap.name)
assert ok
live_after = cj.load_jobs()
live_ids = {j["id"] for j in live_after}
assert "keep-me" in live_ids
assert "delete-me" not in live_ids # not resurrected
def test_rollback_leaves_new_jobs_untouched(backup_env):
"""Jobs created AFTER the snapshot must pass through rollback unchanged."""
cb = backup_env["cb"]
home = backup_env["home"]
_write_skill(backup_env["skills"], "alpha")
_write_cron_jobs(home, [
{"id": "original", "name": "o", "schedule": "every 1h", "skills": ["alpha"]},
])
snap = cb.snapshot_skills(reason="pre-curator-run")
cj = _reload_cron_jobs(home)
jobs = cj.load_jobs()
jobs.append({"id": "new-after-snapshot", "name": "new",
"schedule": "every 15m", "skills": ["brand-new-skill"]})
cj.save_jobs(jobs)
ok, _, _ = cb.rollback(backup_id=snap.name)
assert ok
live = cj.load_jobs()
by_id = {j["id"]: j for j in live}
assert "new-after-snapshot" in by_id
# New job's fields completely preserved
assert by_id["new-after-snapshot"]["skills"] == ["brand-new-skill"]
assert by_id["new-after-snapshot"]["schedule"] == "every 15m"
def test_rollback_with_snapshot_missing_cron_succeeds(backup_env):
"""Older snapshots (created before this feature shipped) have no
cron-jobs.json. Rollback must still restore the skills tree and not
error out."""
cb = backup_env["cb"]
home = backup_env["home"]
_write_skill(backup_env["skills"], "alpha")
# No cron/jobs.json at snapshot time — simulates a pre-feature snapshot
snap = cb.snapshot_skills(reason="test")
assert snap is not None
assert not (snap / cb.CRON_JOBS_FILENAME).exists()
# Later the user created a cron job
_write_cron_jobs(home, [
{"id": "later-job", "name": "l", "schedule": "every 1h", "skills": ["x"]},
])
ok, msg, _ = cb.rollback(backup_id=snap.name)
# Main rollback still succeeds; cron report notes the missing file.
assert ok, msg
# Jobs.json untouched (nothing to restore from)
cj = _reload_cron_jobs(home)
jobs = cj.load_jobs()
assert jobs[0]["id"] == "later-job"
assert jobs[0]["skills"] == ["x"]
def test_restore_cron_skill_links_standalone(backup_env):
"""Unit-level test on _restore_cron_skill_links without the full rollback.
Verifies the report structure carefully."""
cb = backup_env["cb"]
home = backup_env["home"]
# Prime a snapshot dir manually with cron-jobs.json
backups_dir = home / "skills" / ".curator_backups" / "fake-id"
backups_dir.mkdir(parents=True)
(backups_dir / cb.CRON_JOBS_FILENAME).write_text(json.dumps([
{"id": "job-1", "name": "one", "skills": ["narrow-a", "narrow-b"]},
{"id": "job-2", "name": "two", "skill": "legacy-single"},
{"id": "job-gone", "name": "deleted", "skills": ["whatever"]},
]), encoding="utf-8")
# Live jobs: job-1 got rewritten, job-2 unchanged, job-gone deleted
_write_cron_jobs(home, [
{"id": "job-1", "name": "one", "skills": ["umbrella"], "schedule": "every 1h"},
{"id": "job-2", "name": "two", "skill": "legacy-single", "schedule": "every 1h"},
{"id": "job-new", "name": "new", "skills": ["x"], "schedule": "every 1h"},
])
_reload_cron_jobs(home)
report = cb._restore_cron_skill_links(backups_dir)
assert report["attempted"] is True
assert report["error"] is None
assert report["unchanged"] == 1 # job-2 matched
assert len(report["restored"]) == 1 # job-1 got restored
assert report["restored"][0]["job_id"] == "job-1"
assert report["restored"][0]["to"]["skills"] == ["narrow-a", "narrow-b"]
assert len(report["skipped_missing"]) == 1
assert report["skipped_missing"][0]["job_id"] == "job-gone"
File diff suppressed because it is too large Load Diff
+435
View File
@@ -0,0 +1,435 @@
"""Tests for the curator per-run report writer (run.json + REPORT.md).
Reports live under ``~/.hermes/logs/curator/{YYYYMMDD-HHMMSS}/`` alongside
the standard log dir, not inside the user's ``skills/`` data directory.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
import pytest
@pytest.fixture
def curator_env(tmp_path, monkeypatch):
"""Isolated HERMES_HOME with a skills/ dir + reset curator module state."""
home = tmp_path / ".hermes"
home.mkdir()
(home / "skills").mkdir()
(home / "logs").mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
import importlib
import hermes_constants
importlib.reload(hermes_constants)
from agent import curator
importlib.reload(curator)
from tools import skill_usage
importlib.reload(skill_usage)
yield {"home": home, "curator": curator, "skill_usage": skill_usage}
def _make_llm_meta(**overrides):
base = {
"final": "short summary of the pass",
"summary": "short summary",
"model": "test-model",
"provider": "test-provider",
"tool_calls": [],
"error": None,
}
base.update(overrides)
return base
def test_reports_root_is_under_logs_not_skills(curator_env):
"""Reports live in logs/curator/, not skills/ — operational telemetry
belongs with the logs, not with user-authored skill data."""
curator = curator_env["curator"]
root = curator._reports_root()
home = curator_env["home"]
# Must be under logs/
assert root == home / "logs" / "curator"
# Must NOT be under skills/
assert "skills" not in root.parts
def test_write_run_report_creates_both_files(curator_env):
"""Each run writes both a run.json (machine) and a REPORT.md (human)."""
curator = curator_env["curator"]
start = datetime.now(timezone.utc)
run_dir = curator._write_run_report(
started_at=start,
elapsed_seconds=12.345,
auto_counts={"checked": 5, "marked_stale": 1, "archived": 0, "reactivated": 0},
auto_summary="1 marked stale",
before_report=[],
before_names=set(),
after_report=[],
llm_meta=_make_llm_meta(),
)
assert run_dir is not None
assert run_dir.is_dir()
assert (run_dir / "run.json").exists()
assert (run_dir / "REPORT.md").exists()
# The directory name is a timestamp under logs/curator/
assert run_dir.parent == curator._reports_root()
def test_run_json_has_expected_shape(curator_env):
"""run.json must carry the machine-readable fields downstream tooling needs."""
curator = curator_env["curator"]
start = datetime.now(timezone.utc)
before_report = [
{"name": "old-thing", "state": "active", "pinned": False},
{"name": "keeper", "state": "active", "pinned": True},
]
after_report = [
{"name": "keeper", "state": "active", "pinned": True},
{"name": "new-umbrella", "state": "active", "pinned": False},
]
run_dir = curator._write_run_report(
started_at=start,
elapsed_seconds=42.0,
auto_counts={"checked": 2, "marked_stale": 0, "archived": 0, "reactivated": 0},
auto_summary="no changes",
before_report=before_report,
before_names={r["name"] for r in before_report},
after_report=after_report,
llm_meta=_make_llm_meta(
final="I consolidated the whole universe.",
tool_calls=[
{"name": "skills_list", "arguments": "{}"},
{"name": "skill_manage", "arguments": '{"action":"create"}'},
{"name": "terminal", "arguments": "mv ..."},
],
),
)
payload = json.loads((run_dir / "run.json").read_text())
# top-level shape
for k in (
"started_at", "duration_seconds", "model", "provider",
"auto_transitions", "counts", "tool_call_counts",
"archived", "added", "state_transitions",
"llm_final", "llm_summary", "llm_error", "tool_calls",
):
assert k in payload, f"missing key: {k}"
# Diff logic
assert payload["archived"] == ["old-thing"]
assert payload["added"] == ["new-umbrella"]
# Counts reflect the diff
assert payload["counts"]["before"] == 2
assert payload["counts"]["after"] == 2
assert payload["counts"]["archived_this_run"] == 1
assert payload["counts"]["added_this_run"] == 1
# Tool call counts are aggregated
assert payload["tool_call_counts"]["skills_list"] == 1
assert payload["tool_call_counts"]["skill_manage"] == 1
assert payload["tool_call_counts"]["terminal"] == 1
assert payload["counts"]["tool_calls_total"] == 3
def test_report_md_is_human_readable(curator_env):
"""REPORT.md should be a valid markdown doc with the key sections visible."""
curator = curator_env["curator"]
start = datetime.now(timezone.utc)
run_dir = curator._write_run_report(
started_at=start,
elapsed_seconds=75.0,
auto_counts={"checked": 10, "marked_stale": 2, "archived": 1, "reactivated": 0},
auto_summary="2 marked stale, 1 archived",
before_report=[{"name": "foo", "state": "active", "pinned": False}],
before_names={"foo"},
after_report=[{"name": "foo-umbrella", "state": "active", "pinned": False}],
llm_meta=_make_llm_meta(
final="Consolidated foo-like skills into foo-umbrella.",
model="claude-opus-4.7",
provider="openrouter",
tool_calls=[
# Evidence that `foo` was absorbed into `foo-umbrella`:
# write_file under foo-umbrella referencing foo.
{
"name": "skill_manage",
"arguments": json.dumps({
"action": "write_file",
"name": "foo-umbrella",
"file_path": "references/foo.md",
"file_content": "# foo\nContent absorbed from the old foo skill.\n",
}),
},
],
),
)
md = (run_dir / "REPORT.md").read_text()
# Structural checks
assert "# Curator run" in md
assert "Auto-transitions" in md
assert "LLM consolidation pass" in md
assert "Recovery" in md
# The model / provider we passed in show up
assert "claude-opus-4.7" in md
assert "openrouter" in md
# The consolidated/added lists are present with clear language
assert "Consolidated into umbrella skills" in md
assert "`foo`" in md
assert "merged into" in md
assert "`foo-umbrella`" in md
assert "New skills this run" in md
# The full LLM final response is included verbatim (no 240-char truncation)
assert "Consolidated foo-like skills into foo-umbrella." in md
def test_same_second_reruns_get_unique_dirs(curator_env):
"""If the curator somehow runs twice in the same second, the second
report still gets its own directory rather than overwriting the first."""
curator = curator_env["curator"]
start = datetime(2026, 4, 29, 5, 33, 34, tzinfo=timezone.utc)
kwargs = dict(
started_at=start,
elapsed_seconds=1.0,
auto_counts={"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0},
auto_summary="no changes",
before_report=[],
before_names=set(),
after_report=[],
llm_meta=_make_llm_meta(),
)
a = curator._write_run_report(**kwargs)
b = curator._write_run_report(**kwargs)
assert a != b
assert a is not None and b is not None
# Second dir has a numeric disambiguator suffix
assert b.name.startswith(a.name)
def test_report_captures_llm_error_and_continues(curator_env):
"""If the LLM pass recorded an error, the report still writes and
surfaces the error prominently."""
curator = curator_env["curator"]
run_dir = curator._write_run_report(
started_at=datetime.now(timezone.utc),
elapsed_seconds=2.0,
auto_counts={"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0},
auto_summary="no changes",
before_report=[],
before_names=set(),
after_report=[],
llm_meta=_make_llm_meta(
error="HTTP 400: No models provided",
final="",
summary="error",
),
)
md = (run_dir / "REPORT.md").read_text()
assert "HTTP 400" in md
payload = json.loads((run_dir / "run.json").read_text())
assert payload["llm_error"] == "HTTP 400: No models provided"
def test_state_transitions_captured_in_report(curator_env):
"""When a skill moves active → stale or stale → archived between
before/after snapshots, the report records it."""
curator = curator_env["curator"]
start = datetime.now(timezone.utc)
before = [{"name": "getting-old", "state": "active", "pinned": False}]
after = [{"name": "getting-old", "state": "stale", "pinned": False}]
run_dir = curator._write_run_report(
started_at=start,
elapsed_seconds=1.0,
auto_counts={"checked": 1, "marked_stale": 1, "archived": 0, "reactivated": 0},
auto_summary="1 marked stale",
before_report=before,
before_names={r["name"] for r in before},
after_report=after,
llm_meta=_make_llm_meta(),
)
payload = json.loads((run_dir / "run.json").read_text())
assert payload["state_transitions"] == [
{"name": "getting-old", "from": "active", "to": "stale"}
]
md = (run_dir / "REPORT.md").read_text()
assert "State transitions" in md
assert "getting-old" in md
assert "active → stale" in md
# ---------------------------------------------------------------------------
# Cron job skill reference rewriting (curator ↔ cron integration)
# ---------------------------------------------------------------------------
#
# When the curator consolidates skill X into umbrella Y during a run, any
# cron job that listed X in its ``skills`` field would fail to load X at
# run time — the scheduler logs a warning and skips it, so the scheduled
# job runs without the instructions it was scheduled to follow. These
# tests verify that _write_run_report calls into cron.jobs to repair
# those references and records what it did in both run.json and
# cron_rewrites.json.
@pytest.fixture
def curator_env_with_cron(curator_env, monkeypatch):
"""Extend curator_env with an initialized + repointed cron.jobs module."""
home = curator_env["home"]
(home / "cron").mkdir(exist_ok=True)
(home / "cron" / "output").mkdir(exist_ok=True)
import importlib
import cron.jobs as jobs_mod
importlib.reload(jobs_mod)
monkeypatch.setattr(jobs_mod, "HERMES_DIR", home)
monkeypatch.setattr(jobs_mod, "CRON_DIR", home / "cron")
monkeypatch.setattr(jobs_mod, "JOBS_FILE", home / "cron" / "jobs.json")
monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", home / "cron" / "output")
return {**curator_env, "jobs": jobs_mod}
def test_curator_rewrites_cron_skills_when_skill_consolidated(curator_env_with_cron):
"""A skill consolidated into an umbrella should be rewritten in any
cron job's skills list; the rewrite should be visible in run.json
and cron_rewrites.json."""
curator = curator_env_with_cron["curator"]
jobs = curator_env_with_cron["jobs"]
# Create a cron job that depends on a soon-to-be-consolidated skill
job = jobs.create_job(
prompt="",
schedule="every 1h",
skills=["foo"],
name="foo-watcher",
)
# Simulate a curator pass that consolidated `foo` → `foo-umbrella`
before = [{"name": "foo", "state": "active", "pinned": False}]
after = [{"name": "foo-umbrella", "state": "active", "pinned": False}]
run_dir = curator._write_run_report(
started_at=datetime.now(timezone.utc),
elapsed_seconds=3.0,
auto_counts={"checked": 1, "marked_stale": 0, "archived": 0, "reactivated": 0},
auto_summary="no changes",
before_report=before,
before_names={"foo"},
after_report=after,
llm_meta=_make_llm_meta(
final="Consolidated foo into foo-umbrella.",
tool_calls=[
{
"name": "skill_manage",
"arguments": json.dumps({
"action": "write_file",
"name": "foo-umbrella",
"file_path": "references/foo.md",
"file_content": "from foo",
}),
},
],
),
)
# Cron job is rewritten on disk
loaded = jobs.get_job(job["id"])
assert loaded["skills"] == ["foo-umbrella"]
assert loaded["skill"] == "foo-umbrella"
# Rewrite is recorded in run.json
payload = json.loads((run_dir / "run.json").read_text())
assert payload["cron_rewrites"]["jobs_updated"] == 1
assert payload["counts"]["cron_jobs_rewritten"] == 1
rewrites = payload["cron_rewrites"]["rewrites"]
assert len(rewrites) == 1
assert rewrites[0]["mapped"] == {"foo": "foo-umbrella"}
# Separate cron_rewrites.json is written for convenience
cron_file = run_dir / "cron_rewrites.json"
assert cron_file.exists()
detail = json.loads(cron_file.read_text())
assert detail["jobs_updated"] == 1
# Markdown surfaces the change
md = (run_dir / "REPORT.md").read_text()
assert "Cron job skill references rewritten" in md
assert "foo-watcher" in md
assert "foo-umbrella" in md
def test_curator_drops_pruned_skill_from_cron_job(curator_env_with_cron):
"""A pruned (no-umbrella) skill should be dropped from the cron
job's skill list entirely — there's no forwarding target."""
curator = curator_env_with_cron["curator"]
jobs = curator_env_with_cron["jobs"]
job = jobs.create_job(
prompt="",
schedule="every 1h",
skills=["keep", "stale-one"],
)
before = [{"name": "stale-one", "state": "active", "pinned": False}]
after: list = [] # stale-one was archived with no target
run_dir = curator._write_run_report(
started_at=datetime.now(timezone.utc),
elapsed_seconds=1.0,
auto_counts={"checked": 1, "marked_stale": 0, "archived": 1, "reactivated": 0},
auto_summary="1 archived",
before_report=before,
before_names={"stale-one"},
after_report=after,
llm_meta=_make_llm_meta(), # no tool calls → classifier marks it pruned
)
loaded = jobs.get_job(job["id"])
assert loaded["skills"] == ["keep"]
payload = json.loads((run_dir / "run.json").read_text())
assert payload["cron_rewrites"]["jobs_updated"] == 1
rewrites = payload["cron_rewrites"]["rewrites"]
assert rewrites[0]["dropped"] == ["stale-one"]
def test_curator_report_has_no_cron_section_when_nothing_changes(curator_env_with_cron):
"""When the curator run doesn't touch any skills, cron jobs are
untouched and cron_rewrites.json is not even written."""
curator = curator_env_with_cron["curator"]
jobs = curator_env_with_cron["jobs"]
jobs.create_job(prompt="", schedule="every 1h", skills=["foo"])
run_dir = curator._write_run_report(
started_at=datetime.now(timezone.utc),
elapsed_seconds=1.0,
auto_counts={"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0},
auto_summary="no changes",
before_report=[{"name": "foo", "state": "active", "pinned": False}],
before_names={"foo"},
after_report=[{"name": "foo", "state": "active", "pinned": False}],
llm_meta=_make_llm_meta(),
)
# No rewrites → no separate file, no section in md
assert not (run_dir / "cron_rewrites.json").exists()
md = (run_dir / "REPORT.md").read_text()
assert "Cron job skill references rewritten" not in md
payload = json.loads((run_dir / "run.json").read_text())
assert payload["cron_rewrites"]["jobs_updated"] == 0
assert payload["counts"]["cron_jobs_rewritten"] == 0
@@ -0,0 +1,93 @@
from types import SimpleNamespace
from agent.agent_init import _merge_custom_provider_extra_body
def test_custom_provider_extra_body_merges_into_request_overrides():
agent = SimpleNamespace(
provider="custom",
model="google/gemma-4-31b-it",
base_url="https://example.test/v1",
request_overrides={"service_tier": "priority"},
)
_merge_custom_provider_extra_body(
agent,
[
{
"name": "gemma",
"base_url": "https://example.test/v1/",
"model": "google/gemma-4-31b-it",
"extra_body": {
"enable_thinking": True,
"reasoning_effort": "high",
},
}
],
)
assert agent.request_overrides == {
"service_tier": "priority",
"extra_body": {
"enable_thinking": True,
"reasoning_effort": "high",
},
}
def test_custom_provider_extra_body_preserves_caller_override():
agent = SimpleNamespace(
provider="custom",
model="google/gemma-4-31b-it",
base_url="https://example.test/v1",
request_overrides={
"extra_body": {
"reasoning_effort": "low",
"caller_only": True,
}
},
)
_merge_custom_provider_extra_body(
agent,
[
{
"name": "gemma",
"base_url": "https://example.test/v1",
"model": "google/gemma-4-31b-it",
"extra_body": {
"enable_thinking": True,
"reasoning_effort": "high",
},
}
],
)
assert agent.request_overrides["extra_body"] == {
"enable_thinking": True,
"reasoning_effort": "low",
"caller_only": True,
}
def test_custom_provider_extra_body_ignores_other_custom_models():
agent = SimpleNamespace(
provider="custom",
model="other-model",
base_url="https://example.test/v1",
request_overrides={},
)
_merge_custom_provider_extra_body(
agent,
[
{
"name": "gemma",
"base_url": "https://example.test/v1",
"model": "google/gemma-4-31b-it",
"extra_body": {"enable_thinking": True},
}
],
)
assert agent.request_overrides == {}
@@ -0,0 +1,242 @@
"""Regression guard: preserve thinking blocks on DeepSeek's /anthropic endpoint.
DeepSeek's ``api.deepseek.com/anthropic`` route speaks the Anthropic Messages
protocol but, when thinking mode is enabled, requires ``thinking`` blocks from
prior assistant turns to round-trip on subsequent requests. The generic
third-party path strips them (signatures are Anthropic-proprietary and other
proxies cannot validate them), so without a DeepSeek-specific carve-out the
next tool-call turn fails with HTTP 400::
The content[].thinking in the thinking mode must be passed back to the
API.
DeepSeek's compatibility matrix lists ``thinking`` as supported but
``redacted_thinking`` and ``cache_control`` on thinking blocks as not
supported. Handling is the same as Kimi's ``/coding`` endpoint: strip
Anthropic-signed blocks (DeepSeek can't validate them) but preserve unsigned
blocks that Hermes synthesises from ``reasoning_content``.
See hermes-agent#16748.
"""
from __future__ import annotations
import pytest
class TestDeepSeekAnthropicPreservesThinking:
"""convert_messages_to_anthropic must replay DeepSeek thinking blocks."""
@pytest.mark.parametrize(
"base_url",
[
"https://api.deepseek.com/anthropic",
"https://api.deepseek.com/anthropic/",
"https://api.deepseek.com/anthropic/v1",
"https://API.DeepSeek.com/anthropic",
],
)
def test_unsigned_thinking_block_survives_replay(self, base_url: str) -> None:
"""Unsigned thinking (synthesised from reasoning_content) must be preserved."""
from agent.anthropic_adapter import convert_messages_to_anthropic
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"reasoning_content": "planning the tool call",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "skill_view", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
]
_system, converted = convert_messages_to_anthropic(
messages, base_url=base_url
)
assistant_msg = next(m for m in converted if m["role"] == "assistant")
thinking_blocks = [
b for b in assistant_msg["content"]
if isinstance(b, dict) and b.get("type") == "thinking"
]
assert len(thinking_blocks) == 1, (
f"DeepSeek /anthropic ({base_url}) must preserve unsigned thinking "
"blocks synthesised from reasoning_content — upstream rejects "
"replayed tool-call messages without them."
)
assert thinking_blocks[0]["thinking"] == "planning the tool call"
# Synthesised block — never has a signature
assert "signature" not in thinking_blocks[0]
def test_unsigned_thinking_preserved_on_non_latest_assistant_turn(self) -> None:
"""DeepSeek validates history across every prior assistant turn, not just last."""
from agent.anthropic_adapter import convert_messages_to_anthropic
messages = [
{"role": "user", "content": "q1"},
{
"role": "assistant",
"reasoning_content": "r1",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
{"role": "user", "content": "q2"},
{
"role": "assistant",
"reasoning_content": "r2",
"tool_calls": [
{
"id": "call_2",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_2", "content": "ok"},
]
_system, converted = convert_messages_to_anthropic(
messages, base_url="https://api.deepseek.com/anthropic"
)
assistants = [m for m in converted if m["role"] == "assistant"]
assert len(assistants) == 2
for assistant, expected in zip(assistants, ("r1", "r2")):
thinking = [
b for b in assistant["content"]
if isinstance(b, dict) and b.get("type") == "thinking"
]
assert len(thinking) == 1
assert thinking[0]["thinking"] == expected
def test_signed_anthropic_thinking_block_is_stripped(self) -> None:
"""Anthropic-signed blocks (that leaked through) must still be stripped.
DeepSeek issues its own signatures and cannot validate Anthropic's —
the strip-signed / keep-unsigned split matches the Kimi policy.
"""
from agent.anthropic_adapter import convert_messages_to_anthropic
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": "anthropic-signed payload",
"signature": "anthropic-sig-xyz",
},
{"type": "text", "text": "hello"},
],
},
{"role": "user", "content": "again"},
]
_system, converted = convert_messages_to_anthropic(
messages, base_url="https://api.deepseek.com/anthropic"
)
assistant_msg = next(m for m in converted if m["role"] == "assistant")
thinking_blocks = [
b for b in assistant_msg["content"]
if isinstance(b, dict) and b.get("type") == "thinking"
]
assert thinking_blocks == [], (
"Signed Anthropic thinking blocks must be stripped on DeepSeek — "
"DeepSeek cannot validate Anthropic-proprietary signatures."
)
def test_cache_control_stripped_from_thinking_block(self) -> None:
"""cache_control must still be stripped even when the block is preserved.
DeepSeek's compatibility matrix lists cache_control on thinking blocks
as ignored — cache markers interfere with signature validation on
upstreams that do check them, so Hermes strips them everywhere.
"""
from agent.anthropic_adapter import convert_messages_to_anthropic
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"reasoning_content": "r1",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
]
# Inject cache_control on the synthesised thinking block after-the-fact
# by running conversion once, mutating, then re-running would be
# indirect. Instead check the simpler invariant: no thinking block in
# the converted output carries cache_control.
_system, converted = convert_messages_to_anthropic(
messages, base_url="https://api.deepseek.com/anthropic"
)
for m in converted:
if not isinstance(m.get("content"), list):
continue
for b in m["content"]:
if isinstance(b, dict) and b.get("type") in {"thinking", "redacted_thinking"}:
assert "cache_control" not in b
def test_openai_compat_deepseek_base_is_not_matched(self) -> None:
"""The OpenAI-compatible ``api.deepseek.com`` base must NOT trigger the
DeepSeek /anthropic branch — it never reaches this adapter, but the
detector should still fail closed so an accidental misuse doesn't
quietly send signed Anthropic blocks to an OpenAI endpoint.
"""
from agent.anthropic_adapter import _is_deepseek_anthropic_endpoint
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com") is False
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com/v1") is False
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com/anthropic") is True
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com/anthropic/v1") is True
def test_non_deepseek_third_party_still_strips_all_thinking(self) -> None:
"""MiniMax and other third-party Anthropic endpoints must keep the
generic strip-all behaviour (they reject unsigned blocks outright).
"""
from agent.anthropic_adapter import convert_messages_to_anthropic
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"reasoning_content": "r1",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
]
_system, converted = convert_messages_to_anthropic(
messages, base_url="https://api.minimax.io/anthropic"
)
assistant_msg = next(m for m in converted if m["role"] == "assistant")
thinking_blocks = [
b for b in assistant_msg["content"]
if isinstance(b, dict) and b.get("type") == "thinking"
]
assert thinking_blocks == [], (
"Non-DeepSeek third-party endpoints must keep the generic "
"strip-all-thinking behaviour — unsigned blocks get rejected."
)
@@ -0,0 +1,27 @@
from __future__ import annotations
from run_agent import AIAgent
def _agent_with_base_url(base_url: str) -> AIAgent:
agent = object.__new__(AIAgent)
agent.base_url = base_url
return agent
def test_direct_openai_url_requires_openai_host():
agent = _agent_with_base_url("https://api.openai.com.example/v1")
assert agent._is_direct_openai_url() is False
def test_direct_openai_url_ignores_path_segment_match():
agent = _agent_with_base_url("https://proxy.example.test/api.openai.com/v1")
assert agent._is_direct_openai_url() is False
def test_direct_openai_url_accepts_native_host():
agent = _agent_with_base_url("https://api.openai.com/v1")
assert agent._is_direct_openai_url() is True
+278
View File
@@ -0,0 +1,278 @@
"""Tests for agent/display.py — build_tool_preview() and inline diff previews."""
import json
import pytest
from unittest.mock import MagicMock
from agent.display import (
build_tool_preview,
capture_local_edit_snapshot,
extract_edit_diff,
get_cute_tool_message,
set_tool_preview_max_len,
_render_inline_unified_diff,
_summarize_rendered_diff_sections,
render_edit_diff_with_delta,
)
@pytest.fixture(autouse=True)
def reset_tool_preview_max_len():
set_tool_preview_max_len(0)
yield
set_tool_preview_max_len(0)
class TestBuildToolPreview:
"""Tests for build_tool_preview defensive handling and normal operation."""
def test_none_args_returns_none(self):
"""PR #453: None args should not crash, should return None."""
assert build_tool_preview("terminal", None) is None
def test_empty_dict_returns_none(self):
"""Empty dict has no keys to preview."""
assert build_tool_preview("terminal", {}) is None
def test_known_tool_with_primary_arg(self):
"""Known tool with its primary arg should return a preview string."""
result = build_tool_preview("terminal", {"command": "ls -la"})
assert result is not None
assert "ls -la" in result
def test_web_search_preview(self):
result = build_tool_preview("web_search", {"query": "hello world"})
assert result is not None
assert "hello world" in result
def test_read_file_preview(self):
result = build_tool_preview("read_file", {"path": "/tmp/test.py", "offset": 1})
assert result is not None
assert "/tmp/test.py" in result
def test_unknown_tool_with_fallback_key(self):
"""Unknown tool but with a recognized fallback key should still preview."""
result = build_tool_preview("custom_tool", {"query": "test query"})
assert result is not None
assert "test query" in result
def test_unknown_tool_no_matching_key(self):
"""Unknown tool with no recognized keys should return None."""
result = build_tool_preview("custom_tool", {"foo": "bar"})
assert result is None
def test_long_value_truncated(self):
"""Preview should truncate long values."""
long_cmd = "a" * 100
result = build_tool_preview("terminal", {"command": long_cmd}, max_len=40)
assert result is not None
assert len(result) <= 43 # max_len + "..."
def test_process_tool_with_none_args(self):
"""Process tool special case should also handle None args."""
assert build_tool_preview("process", None) is None
def test_process_tool_normal(self):
result = build_tool_preview("process", {"action": "poll", "session_id": "abc123"})
assert result is not None
assert "poll" in result
def test_todo_tool_read(self):
result = build_tool_preview("todo", {"merge": False})
assert result is not None
assert "reading" in result
def test_todo_tool_with_todos(self):
result = build_tool_preview("todo", {"todos": [{"id": "1", "content": "test", "status": "pending"}]})
assert result is not None
assert "1 task" in result
def test_memory_tool_add(self):
result = build_tool_preview("memory", {"action": "add", "target": "user", "content": "test note"})
assert result is not None
assert "user" in result
def test_memory_replace_missing_old_text_marked(self):
# Avoid empty quotes "" in the preview when old_text is missing/None.
result = build_tool_preview("memory", {"action": "replace", "target": "memory"})
assert result == '~memory: "<missing old_text>"'
result = build_tool_preview("memory", {"action": "remove", "target": "memory", "old_text": None})
assert result == '-memory: "<missing old_text>"'
def test_session_search_preview(self):
result = build_tool_preview("session_search", {"query": "find something"})
assert result is not None
assert "find something" in result
def test_false_like_args_zero(self):
"""Non-dict falsy values should return None, not crash."""
assert build_tool_preview("terminal", 0) is None
assert build_tool_preview("terminal", "") is None
assert build_tool_preview("terminal", []) is None
class TestCuteToolMessagePreviewLength:
def test_terminal_preview_unlimited_when_config_is_zero(self):
set_tool_preview_max_len(0)
command = "curl -s http://localhost:9222/json/list | jq -r '.[] | select(.type==\"page\")' | head -5"
line = get_cute_tool_message("terminal", {"command": command}, 0.1)
assert command in line
assert "..." not in line
def test_terminal_preview_uses_positive_configured_limit(self):
set_tool_preview_max_len(80)
command = "curl -s http://localhost:9222/json/list | jq -r '.[] | select(.type==\"page\")' | head -5"
line = get_cute_tool_message("terminal", {"command": command}, 0.1)
assert command[:77] in line
assert "..." in line
assert "head -5" not in line
def test_search_files_preview_uses_positive_configured_limit_not_default(self):
set_tool_preview_max_len(80)
pattern = "function.formatToolCall.context.preview.compactPreview.maxLength.truncate"
line = get_cute_tool_message("search_files", {"pattern": pattern}, 0.1)
assert pattern in line
assert "..." not in line
def test_path_preview_uses_positive_configured_limit_not_default(self):
set_tool_preview_max_len(80)
path = "/tmp/hermes-test-preview-length/deeply/nested/path/test-output.txt"
line = get_cute_tool_message("read_file", {"path": path}, 0.1)
assert path in line
assert "..." not in line
def test_write_file_lint_error_result_is_not_marked_failed(self):
result = json.dumps({
"bytes_written": 12,
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
})
line = get_cute_tool_message("write_file", {"path": "/tmp/a.py"}, 0.1, result=result)
assert "[error]" not in line
def test_patch_lsp_diagnostics_result_is_not_marked_failed(self):
result = json.dumps({
"success": True,
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
})
line = get_cute_tool_message("patch", {"path": "/tmp/a.py"}, 0.1, result=result)
assert "[error]" not in line
class TestEditDiffPreview:
def test_extract_edit_diff_for_patch(self):
diff = extract_edit_diff("patch", '{"success": true, "diff": "--- a/x\\n+++ b/x\\n"}')
assert diff is not None
assert "+++ b/x" in diff
def test_render_inline_unified_diff_colors_added_and_removed_lines(self):
rendered = _render_inline_unified_diff(
"--- a/cli.py\n"
"+++ b/cli.py\n"
"@@ -1,2 +1,2 @@\n"
"-old line\n"
"+new line\n"
" context\n"
)
assert "a/cli.py" in rendered[0]
assert "b/cli.py" in rendered[0]
assert any("old line" in line for line in rendered)
assert any("new line" in line for line in rendered)
assert any("48;2;" in line for line in rendered)
def test_extract_edit_diff_ignores_non_edit_tools(self):
assert extract_edit_diff("web_search", '{"diff": "--- a\\n+++ b\\n"}') is None
def test_extract_edit_diff_uses_local_snapshot_for_write_file(self, tmp_path):
target = tmp_path / "note.txt"
target.write_text("old\n", encoding="utf-8")
snapshot = capture_local_edit_snapshot("write_file", {"path": str(target)})
target.write_text("new\n", encoding="utf-8")
diff = extract_edit_diff(
"write_file",
'{"bytes_written": 4}',
function_args={"path": str(target)},
snapshot=snapshot,
)
assert diff is not None
assert "--- a/" in diff
assert "+++ b/" in diff
assert "-old" in diff
assert "+new" in diff
def test_render_edit_diff_with_delta_invokes_printer(self):
printer = MagicMock()
rendered = render_edit_diff_with_delta(
"patch",
'{"diff": "--- a/x\\n+++ b/x\\n@@ -1 +1 @@\\n-old\\n+new\\n"}',
print_fn=printer,
)
assert rendered is True
assert printer.call_count >= 2
calls = [call.args[0] for call in printer.call_args_list]
assert any("a/x" in line and "b/x" in line for line in calls)
assert any("old" in line for line in calls)
assert any("new" in line for line in calls)
def test_render_edit_diff_with_delta_skips_without_diff(self):
rendered = render_edit_diff_with_delta(
"patch",
'{"success": true}',
)
assert rendered is False
def test_render_edit_diff_with_delta_handles_renderer_errors(self, monkeypatch):
printer = MagicMock()
monkeypatch.setattr("agent.display._summarize_rendered_diff_sections", MagicMock(side_effect=RuntimeError("boom")))
rendered = render_edit_diff_with_delta(
"patch",
'{"diff": "--- a/x\\n+++ b/x\\n"}',
print_fn=printer,
)
assert rendered is False
assert printer.call_count == 0
def test_summarize_rendered_diff_sections_truncates_large_diff(self):
diff = "--- a/x.py\n+++ b/x.py\n" + "".join(f"+line{i}\n" for i in range(120))
rendered = _summarize_rendered_diff_sections(diff, max_lines=20)
assert len(rendered) == 21
assert "omitted" in rendered[-1]
def test_summarize_rendered_diff_sections_limits_file_count(self):
diff = "".join(
f"--- a/file{i}.py\n+++ b/file{i}.py\n+line{i}\n"
for i in range(8)
)
rendered = _summarize_rendered_diff_sections(diff, max_files=3, max_lines=50)
assert any("a/file0.py" in line for line in rendered)
assert any("a/file1.py" in line for line in rendered)
assert any("a/file2.py" in line for line in rendered)
assert not any("a/file7.py" in line for line in rendered)
assert "additional file" in rendered[-1]
+123
View File
@@ -0,0 +1,123 @@
"""Tests for get_tool_emoji in agent/display.py — skin + registry integration."""
from unittest.mock import patch as mock_patch, MagicMock
from agent.display import get_tool_emoji
class TestGetToolEmoji:
"""Verify the skin → registry → fallback resolution chain."""
def test_returns_registry_emoji_when_no_skin(self):
"""Registry-registered emoji is used when no skin is active."""
mock_registry = MagicMock()
mock_registry.get_emoji.return_value = "🎨"
with mock_patch("agent.display._get_skin", return_value=None), \
mock_patch("agent.display.registry", mock_registry, create=True):
# Need to patch the import inside get_tool_emoji
pass
# Direct test: patch the lazy import path
with mock_patch("agent.display._get_skin", return_value=None):
# get_tool_emoji will try to import registry — mock that
mock_reg = MagicMock()
mock_reg.get_emoji.return_value = "📖"
with mock_patch.dict("sys.modules", {}):
import sys
# Patch tools.registry module
mock_module = MagicMock()
mock_module.registry = mock_reg
with mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
result = get_tool_emoji("read_file")
assert result == "📖"
def test_skin_override_takes_precedence(self):
"""Skin tool_emojis override registry defaults."""
skin = MagicMock()
skin.tool_emojis = {"terminal": "⚔"}
with mock_patch("agent.display._get_skin", return_value=skin):
result = get_tool_emoji("terminal")
assert result == "⚔"
def test_skin_empty_dict_falls_through(self):
"""Empty skin tool_emojis falls through to registry."""
skin = MagicMock()
skin.tool_emojis = {}
mock_reg = MagicMock()
mock_reg.get_emoji.return_value = "💻"
import sys
mock_module = MagicMock()
mock_module.registry = mock_reg
with mock_patch("agent.display._get_skin", return_value=skin), \
mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
result = get_tool_emoji("terminal")
assert result == "💻"
def test_fallback_default(self):
"""When neither skin nor registry has an emoji, use the default."""
skin = MagicMock()
skin.tool_emojis = {}
mock_reg = MagicMock()
mock_reg.get_emoji.return_value = ""
import sys
mock_module = MagicMock()
mock_module.registry = mock_reg
with mock_patch("agent.display._get_skin", return_value=skin), \
mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
result = get_tool_emoji("unknown_tool")
assert result == "⚡"
def test_custom_default(self):
"""Custom default is returned when nothing matches."""
with mock_patch("agent.display._get_skin", return_value=None):
mock_reg = MagicMock()
mock_reg.get_emoji.return_value = ""
import sys
mock_module = MagicMock()
mock_module.registry = mock_reg
with mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
result = get_tool_emoji("x", default="⚙️")
assert result == "⚙️"
def test_skin_override_only_for_matching_tool(self):
"""Skin override for one tool doesn't affect others."""
skin = MagicMock()
skin.tool_emojis = {"terminal": "⚔"}
mock_reg = MagicMock()
mock_reg.get_emoji.return_value = "🔍"
import sys
mock_module = MagicMock()
mock_module.registry = mock_reg
with mock_patch("agent.display._get_skin", return_value=skin), \
mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
assert get_tool_emoji("terminal") == "⚔" # skin override
assert get_tool_emoji("web_search") == "🔍" # registry fallback
class TestSkinConfigToolEmojis:
"""Verify SkinConfig handles tool_emojis field correctly."""
def test_skin_config_has_tool_emojis_field(self):
from hermes_cli.skin_engine import SkinConfig
skin = SkinConfig(name="test")
assert skin.tool_emojis == {}
def test_skin_config_accepts_tool_emojis(self):
from hermes_cli.skin_engine import SkinConfig
emojis = {"terminal": "⚔", "web_search": "🔮"}
skin = SkinConfig(name="test", tool_emojis=emojis)
assert skin.tool_emojis == emojis
def test_build_skin_config_includes_tool_emojis(self):
from hermes_cli.skin_engine import _build_skin_config
data = {
"name": "custom",
"tool_emojis": {"terminal": "🗡️", "patch": "⚒️"},
}
skin = _build_skin_config(data)
assert skin.tool_emojis == {"terminal": "🗡️", "patch": "⚒️"}
def test_build_skin_config_empty_tool_emojis_default(self):
from hermes_cli.skin_engine import _build_skin_config
data = {"name": "minimal"}
skin = _build_skin_config(data)
assert skin.tool_emojis == {}
+242
View File
@@ -0,0 +1,242 @@
"""Tests for get_cute_tool_message todo progress display.
Verifies the completion status rendering (done/total ✓) on all three
todo tool call paths: read, create (merge=False), update (merge=True).
"""
import json
from agent.display import get_cute_tool_message
def _todo_result(total: int, completed: int) -> str:
"""Build a fake todo_tool return value."""
return json.dumps({
"todos": [],
"summary": {
"total": total,
"pending": total - completed,
"in_progress": 0,
"completed": completed,
"cancelled": 0,
},
})
class TestTodoRead:
"""get_cute_tool_message(…, result=…) when todos_arg is None (read path)."""
def test_read_no_result(self):
msg = get_cute_tool_message("todo", {}, 0.5)
assert "reading tasks" in msg
assert "0.5s" in msg
def test_read_with_progress(self):
msg = get_cute_tool_message("todo", {}, 0.5,
result=_todo_result(4, 2))
assert "2/4" in msg
assert "task(s)" in msg
def test_read_all_done(self):
msg = get_cute_tool_message("todo", {}, 0.5,
result=_todo_result(4, 4))
assert "4/4" in msg
assert "task(s)" in msg
def test_read_zero_total(self):
"""Edge case: empty todo list returns summary with total=0."""
msg = get_cute_tool_message("todo", {}, 0.5,
result=_todo_result(0, 0))
assert "reading tasks" in msg
def test_read_invalid_result_fallback(self):
"""Garbage result should not crash; fall back to reading tasks."""
msg = get_cute_tool_message("todo", {}, 0.5, result="not json")
assert "reading tasks" in msg
def test_read_result_missing_summary(self):
msg = get_cute_tool_message("todo", {}, 0.5,
result='{"todos": []}')
assert "reading tasks" in msg
class TestTodoCreate:
"""get_cute_tool_message when merge=False (new plan creation)."""
def test_create_default(self):
"""Brand-new plan: all pending, no result — plain count."""
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "content": "x", "status": "pending"},
]}, 0.3)
assert "1 task(s)" in msg
assert "0.3s" in msg
assert "/" not in msg # no progress fraction
def test_create_multiple(self):
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "content": "x", "status": "pending"},
{"id": "b", "content": "y", "status": "pending"},
{"id": "c", "content": "z", "status": "pending"},
]}, 0.2)
assert "3 task(s)" in msg
def test_create_with_result_shows_progress_when_done(self):
"""Even on create, if result has completed tasks show it."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "content": "x", "status": "completed"}]},
0.4,
result=_todo_result(1, 1))
assert "1/1" in msg
assert "task(s)" in msg
def test_create_with_result_zero_done(self):
"""New plan with 0 done — plain count, no progress fraction."""
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "content": "x", "status": "pending"},
{"id": "b", "content": "y", "status": "pending"},
]},
0.3,
result=_todo_result(2, 0))
assert "2 task(s)" in msg
assert "/" not in msg
class TestTodoUpdate:
"""get_cute_tool_message when merge=True (incremental update)."""
def test_update_no_result(self):
"""No result available — plain update N task(s)."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True}, 0.5)
assert "update 1 task(s)" in msg
def test_update_partial_progress(self):
"""1/4 tasks completed — show fraction with checkmark."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.5,
result=_todo_result(4, 1))
assert "update" in msg
assert "1/4" in msg
assert "✓" in msg
def test_update_halfway(self):
"""2/4 — midpoint progress."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "b", "status": "in_progress"}],
"merge": True},
0.7,
result=_todo_result(4, 2))
assert "2/4" in msg
assert "✓" in msg
def test_update_all_completed(self):
"""4/4 — full checkmark."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "d", "status": "completed"}],
"merge": True},
0.2,
result=_todo_result(4, 4))
assert "4/4" in msg
assert "✓" in msg
def test_update_zero_done(self):
"""No completed tasks yet — plain update N task(s)."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "pending"}],
"merge": True},
0.3,
result=_todo_result(3, 0))
assert "update 1 task(s)" in msg
assert "✓" not in msg
assert "/" not in msg # no progress fraction when done=0
def test_update_invalid_result_fallback(self):
"""Bad JSON result — fall back to plain update N task(s)."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.6,
result="{broken")
assert "update 1 task(s)" in msg
assert "✓" not in msg
def test_update_result_missing_summary(self):
"""Result no summary key — fall back to plain update."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.4,
result='{"todos": []}')
assert "update 1 task(s)" in msg
assert "✓" not in msg
def test_update_total_not_in_summary(self):
"""Result summary missing total key."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.3,
result=json.dumps({"summary": {"completed": 2}}))
assert "update 1 task(s)" in msg
assert "✓" not in msg
def test_update_multiple_tasks_in_line(self):
"""Update line with several tasks in the update request."""
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "status": "completed"},
{"id": "b", "status": "in_progress"},
], "merge": True},
0.5,
result=_todo_result(5, 3))
assert "update" in msg
assert "3/5" in msg
assert "✓" in msg
class TestTodoEdgeCases:
"""Boundary cases that should not crash."""
def test_merge_default_value(self):
"""merge defaults to False in function signature, should be False when absent."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "content": "x", "status": "pending"}]},
1.0)
assert "1 task(s)" in msg
def test_duration_formatting(self):
"""Duration formatting works correctly."""
msg = get_cute_tool_message("todo", {}, 0.123)
assert "0.1s" in msg
msg = get_cute_tool_message("todo", {}, 1.0)
assert "1.0s" in msg
msg = get_cute_tool_message("todo", {}, 123.456)
assert "123.5s" in msg
def test_large_task_count(self):
"""Many tasks should not break formatting."""
many = [{"id": str(i), "content": "x", "status": "pending"} for i in range(50)]
msg = get_cute_tool_message("todo", {"todos": many}, 0.5)
assert "50 task(s)" in msg
def test_read_with_no_args_and_no_result(self):
"""Completely empty call."""
msg = get_cute_tool_message("todo", {}, 0.0)
assert "reading tasks" in msg
class TestTodoSkinIntegration:
"""Verify the skin prefix is applied to todo messages too.
This uses the same pattern as test_skin_engine test_tool_message_uses_skin_prefix.
"""
def test_default_skin_prefix(self):
msg = get_cute_tool_message("todo", {}, 0.5)
assert msg.startswith("┊")
+184
View File
@@ -0,0 +1,184 @@
"""Tests for _detect_tool_failure + _trim_error + get_cute_tool_message
inline failure suffix rendering.
Covers the user-visible promise: when a tool fails, the CLI shows a short,
specific reason in square brackets at the end of the completion line —
not a generic "[error]".
"""
import json
from agent.display import (
_detect_tool_failure,
_trim_error,
_ERROR_SUFFIX_MAX_LEN,
get_cute_tool_message,
)
class TestTrimError:
"""The helper that shrinks an error message for inline display."""
def test_short_message_unchanged(self):
assert _trim_error("nope") == "nope"
def test_whitespace_stripped(self):
assert _trim_error(" bad input ") == "bad input"
def test_long_message_truncated_to_cap(self):
msg = "x" * 200
trimmed = _trim_error(msg)
assert len(trimmed) <= _ERROR_SUFFIX_MAX_LEN
assert trimmed.endswith("...")
def test_file_not_found_path_collapsed_to_filename(self):
long_path = "File not found: /home/teknium/.hermes/hermes-agent/very/deep/path/foo.py"
assert _trim_error(long_path) == "File not found: foo.py"
def test_file_not_found_already_short_unchanged(self):
assert _trim_error("File not found: foo.py") == "File not found: foo.py"
def test_file_not_found_relative_path_unchanged(self):
# Without a slash there's no path to trim.
assert _trim_error("File not found: foo.py") == "File not found: foo.py"
class TestDetectToolFailureTerminal:
"""terminal: non-zero exit_code is the canonical failure signal."""
def test_success_returns_no_suffix(self):
result = json.dumps({"output": "ok\n", "exit_code": 0})
assert _detect_tool_failure("terminal", result) == (False, "")
def test_nonzero_exit_with_no_error_shows_exit_code(self):
result = json.dumps({"output": "", "exit_code": 1})
is_failure, suffix = _detect_tool_failure("terminal", result)
assert is_failure is True
assert suffix == " [exit 1]"
def test_nonzero_exit_with_error_shows_message(self):
result = json.dumps({
"output": "",
"exit_code": 127,
"error": "ls: cannot access 'foo': No such file or directory",
})
is_failure, suffix = _detect_tool_failure("terminal", result)
assert is_failure is True
assert "cannot access" in suffix
# Trimmed to the cap, in brackets
assert suffix.startswith(" [")
assert suffix.endswith("]")
def test_malformed_json_returns_no_suffix(self):
# Terminal is special: only exit_code matters. Malformed JSON should
# not crash and should not be flagged as failure.
assert _detect_tool_failure("terminal", "not json") == (False, "")
def test_none_result_returns_no_suffix(self):
assert _detect_tool_failure("terminal", None) == (False, "")
class TestDetectToolFailureMemory:
"""memory: 'full' is distinct from real errors."""
def test_memory_full_returns_full_suffix(self):
result = json.dumps({"success": False, "error": "would exceed the limit"})
assert _detect_tool_failure("memory", result) == (True, " [full]")
def test_memory_other_error_returns_specific_message(self):
# An error that's NOT a "full" overflow falls through to the
# structured-error path and surfaces the actual message.
result = json.dumps({"success": False, "error": "invalid action: zap"})
is_failure, suffix = _detect_tool_failure("memory", result)
assert is_failure is True
assert "invalid action" in suffix
class TestDetectToolFailureStructured:
"""Generic path: any tool that returns {"error": ...} JSON."""
def test_read_file_error_surfaced(self):
result = json.dumps({
"path": "/nope/missing.py",
"success": False,
"error": "File not found: /nope/missing.py",
})
is_failure, suffix = _detect_tool_failure("read_file", result)
assert is_failure is True
# _trim_error reduces the path to the basename.
assert suffix == " [File not found: missing.py]"
def test_error_without_success_key_still_flagged(self):
# Some tools return {"error": "..."} with no explicit success flag.
result = json.dumps({"error": "remote unavailable"})
is_failure, suffix = _detect_tool_failure("web_search", result)
assert is_failure is True
assert suffix == " [remote unavailable]"
def test_message_field_only_with_success_false_flagged(self):
# When success is False and only 'message' is set, surface it.
result = json.dumps({"success": False, "message": "rate limited"})
is_failure, suffix = _detect_tool_failure("web_search", result)
assert is_failure is True
assert "rate limited" in suffix
def test_successful_result_not_flagged(self):
result = json.dumps({"success": True, "data": "hello"})
assert _detect_tool_failure("web_search", result) == (False, "")
def test_dict_without_error_or_success_uses_generic_heuristic(self):
# Plain successful dict — should pass through the generic
# heuristic which only fires on the string "Error" / '"error"' / etc.
result = json.dumps({"data": "hello"})
is_failure, _ = _detect_tool_failure("web_search", result)
assert is_failure is False
class TestGetCuteToolMessageFailureSuffix:
"""End-to-end: failure suffix is appended by get_cute_tool_message."""
def test_read_file_failure_suffix_appended(self):
fail = json.dumps({
"path": "/etc/missing",
"success": False,
"error": "File not found: /etc/missing",
})
line = get_cute_tool_message("read_file", {"path": "/etc/missing"}, 0.1, result=fail)
assert "[File not found: missing]" in line
def test_terminal_exit_only_suffix(self):
fail = json.dumps({"output": "", "exit_code": 2})
line = get_cute_tool_message("terminal", {"command": "false"}, 0.1, result=fail)
assert "[exit 2]" in line
def test_terminal_with_stderr_uses_message(self):
fail = json.dumps({
"output": "",
"exit_code": 127,
"error": "command not found: notathing",
})
line = get_cute_tool_message("terminal", {"command": "notathing"}, 0.1, result=fail)
assert "command not found" in line
# No '[exit 127]' tag when we have a specific message
assert "exit 127" not in line
def test_memory_full_suffix(self):
fail = json.dumps({"success": False, "error": "would exceed the limit"})
line = get_cute_tool_message(
"memory",
{"action": "add", "target": "memory", "content": "x"},
0.05,
result=fail,
)
assert "[full]" in line
def test_success_has_no_suffix(self):
ok = json.dumps({"success": True, "data": "hi"})
line = get_cute_tool_message("web_search", {"query": "hi"}, 0.2, result=ok)
assert "[" not in line.split("0.2s", 1)[1]
def test_no_result_has_no_suffix(self):
# No result passed at all — display function should not invent a
# failure suffix.
line = get_cute_tool_message("terminal", {"command": "ls"}, 0.2)
assert "[" not in line.split("0.2s", 1)[1]
File diff suppressed because it is too large Load Diff
+156
View File
@@ -0,0 +1,156 @@
"""Tests for external skill directories (skills.external_dirs config)."""
import json
import os
from unittest.mock import patch
import pytest
@pytest.fixture
def external_skills_dir(tmp_path):
"""Create a temp dir with a sample external skill."""
ext_dir = tmp_path / "external-skills"
skill_dir = ext_dir / "my-external-skill"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\nname: my-external-skill\ndescription: A skill from an external directory\n---\n\n# My External Skill\n\nDo external things.\n"
)
return ext_dir
@pytest.fixture
def hermes_home(tmp_path):
"""Create a minimal HERMES_HOME with config."""
home = tmp_path / ".hermes"
home.mkdir()
(home / "skills").mkdir()
return home
class TestGetExternalSkillsDirs:
def test_empty_config(self, hermes_home):
(hermes_home / "config.yaml").write_text("skills:\n external_dirs: []\n")
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert result == []
def test_nonexistent_dir_skipped(self, hermes_home):
(hermes_home / "config.yaml").write_text(
"skills:\n external_dirs:\n - /nonexistent/path\n"
)
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert result == []
def test_valid_dir_returned(self, hermes_home, external_skills_dir):
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
)
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert len(result) == 1
assert result[0] == external_skills_dir.resolve()
def test_duplicate_dirs_deduplicated(self, hermes_home, external_skills_dir):
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {external_skills_dir}\n - {external_skills_dir}\n"
)
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert len(result) == 1
def test_local_skills_dir_excluded(self, hermes_home):
local_skills = hermes_home / "skills"
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {local_skills}\n"
)
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert result == []
def test_no_config_file(self, hermes_home):
# No config.yaml at all
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert result == []
def test_string_value_converted_to_list(self, hermes_home, external_skills_dir):
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs: {external_skills_dir}\n"
)
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_external_skills_dirs
result = get_external_skills_dirs()
assert len(result) == 1
class TestGetAllSkillsDirs:
def test_local_always_first(self, hermes_home, external_skills_dir):
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
)
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
from agent.skill_utils import get_all_skills_dirs
result = get_all_skills_dirs()
assert result[0] == hermes_home / "skills"
assert result[1] == external_skills_dir.resolve()
class TestExternalSkillsInFindAll:
def test_external_skills_found(self, hermes_home, external_skills_dir):
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
)
local_skills = hermes_home / "skills"
with (
patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}),
patch("tools.skills_tool.SKILLS_DIR", local_skills),
):
from tools.skills_tool import _find_all_skills
skills = _find_all_skills()
names = [s["name"] for s in skills]
assert "my-external-skill" in names
def test_local_takes_precedence(self, hermes_home, external_skills_dir):
"""If the same skill name exists locally and externally, local wins."""
local_skills = hermes_home / "skills"
local_skill = local_skills / "my-external-skill"
local_skill.mkdir(parents=True)
(local_skill / "SKILL.md").write_text(
"---\nname: my-external-skill\ndescription: Local version\n---\n\nLocal.\n"
)
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
)
with (
patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}),
patch("tools.skills_tool.SKILLS_DIR", local_skills),
):
from tools.skills_tool import _find_all_skills
skills = _find_all_skills()
matching = [s for s in skills if s["name"] == "my-external-skill"]
assert len(matching) == 1
assert matching[0]["description"] == "Local version"
class TestExternalSkillView:
def test_skill_view_finds_external(self, hermes_home, external_skills_dir):
(hermes_home / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
)
local_skills = hermes_home / "skills"
with (
patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}),
patch("tools.skills_tool.SKILLS_DIR", local_skills),
):
from tools.skills_tool import skill_view
result = json.loads(skill_view("my-external-skill"))
assert result["success"] is True
assert "external things" in result["content"]
@@ -0,0 +1,148 @@
"""Guards for ``get_external_skills_dirs`` mtime-based memo.
``get_external_skills_dirs()`` is called once per skill during banner
construction and tool registration — on a typical install that's 120+
calls. Without caching, each call re-reads + YAML-parses the full
config.yaml (~85ms each, 10+ seconds total). This test pins the
behavior: first call parses, subsequent calls return cached result,
cache invalidates when config.yaml's mtime changes.
"""
from __future__ import annotations
import os
from pathlib import Path
from unittest.mock import patch
import pytest
from agent import skill_utils
from agent.skill_utils import (
_external_dirs_cache_clear,
get_external_skills_dirs,
)
@pytest.fixture
def hermes_home_with_config(tmp_path, monkeypatch):
"""Isolated ``~/.hermes/`` with a config.yaml referencing one external dir."""
home = tmp_path / ".hermes"
home.mkdir()
external = tmp_path / "external_skills"
external.mkdir()
config = home / "config.yaml"
config.write_text(
"skills:\n"
f" external_dirs:\n"
f" - {external}\n",
encoding="utf-8",
)
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
_external_dirs_cache_clear()
yield home, external, config
_external_dirs_cache_clear()
def test_returns_configured_external_dir(hermes_home_with_config):
_home, external, _cfg = hermes_home_with_config
result = get_external_skills_dirs()
assert result == [external.resolve()]
def test_cache_reuses_result_without_reparsing(hermes_home_with_config):
"""Subsequent calls hit the cache and skip YAML parsing entirely."""
_home, _external, _cfg = hermes_home_with_config
# Prime cache
get_external_skills_dirs()
# Patch yaml_load to raise — if cache works, it's never called again.
with patch.object(
skill_utils,
"yaml_load",
side_effect=AssertionError("yaml_load should not run on cache hit"),
):
# Many calls, none should trigger the patched yaml_load.
for _ in range(100):
get_external_skills_dirs()
def test_cache_invalidates_on_mtime_change(hermes_home_with_config):
"""A config.yaml edit invalidates the cache on the next call."""
_home, external, config = hermes_home_with_config
other = external.parent / "other_skills"
other.mkdir()
# Prime cache with original contents.
first = get_external_skills_dirs()
assert first == [external.resolve()]
# Rewrite config; bump mtime forward explicitly so filesystems with
# coarse mtime granularity still register the change on fast test
# systems.
config.write_text(
"skills:\n"
f" external_dirs:\n"
f" - {other}\n",
encoding="utf-8",
)
stat = config.stat()
future = stat.st_atime + 10
os.utime(config, (future, future))
second = get_external_skills_dirs()
assert second == [other.resolve()]
def test_returns_empty_when_config_missing(tmp_path, monkeypatch):
"""No config file → empty list, cached as empty."""
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
_external_dirs_cache_clear()
assert get_external_skills_dirs() == []
def test_returned_list_is_a_copy(hermes_home_with_config):
"""Callers can't poison the cache by mutating the returned list."""
first = get_external_skills_dirs()
first.append(Path("/tmp/should-not-persist"))
second = get_external_skills_dirs()
assert Path("/tmp/should-not-persist") not in second
def test_cache_key_is_per_config_path(tmp_path, monkeypatch):
"""Two different HERMES_HOMEs keep separate cache entries."""
home_a = tmp_path / "home_a" / ".hermes"
home_a.mkdir(parents=True)
ext_a = tmp_path / "ext_a"
ext_a.mkdir()
(home_a / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {ext_a}\n", encoding="utf-8"
)
home_b = tmp_path / "home_b" / ".hermes"
home_b.mkdir(parents=True)
ext_b = tmp_path / "ext_b"
ext_b.mkdir()
(home_b / "config.yaml").write_text(
f"skills:\n external_dirs:\n - {ext_b}\n", encoding="utf-8"
)
_external_dirs_cache_clear()
monkeypatch.setenv("HERMES_HOME", str(home_a))
assert get_external_skills_dirs() == [ext_a.resolve()]
monkeypatch.setenv("HERMES_HOME", str(home_b))
assert get_external_skills_dirs() == [ext_b.resolve()]
# And switching back still works — both entries coexist in the cache.
monkeypatch.setenv("HERMES_HOME", str(home_a))
assert get_external_skills_dirs() == [ext_a.resolve()]
+148
View File
@@ -0,0 +1,148 @@
"""Tests for agent/file_safety.py read guards — env file blocking.
Run with: python -m pytest tests/agent/test_file_safety.py -v
"""
import os
from unittest.mock import patch
import pytest
from agent.file_safety import (
_BLOCKED_PROJECT_ENV_BASENAMES,
get_read_block_error,
)
# ---------------------------------------------------------------------------
# Project-local .env file blocking (issue #20734)
# ---------------------------------------------------------------------------
class TestEnvFileReadBlocking:
"""Secret-bearing .env files must be blocked by get_read_block_error."""
@pytest.mark.parametrize("basename", [
".env",
".env.local",
".env.development",
".env.production",
".env.test",
".env.staging",
".envrc",
])
def test_blocked_env_basenames(self, basename):
"""All secret-bearing .env basenames are blocked regardless of directory."""
path = f"/tmp/project/{basename}"
error = get_read_block_error(path)
assert error is not None, f"{basename} should be blocked"
assert "Access denied" in error
assert "secret-bearing" in error.lower() or "environment file" in error.lower()
def test_blocked_env_in_subdirectory(self):
"""Nested .env files are also blocked."""
error = get_read_block_error("/home/user/app/services/api/.env.production")
assert error is not None
def test_blocked_env_absolute_path(self):
"""Absolute paths to .env files are blocked."""
error = get_read_block_error("/opt/myapp/.env")
assert error is not None
def test_allowed_env_example(self):
""""The .env.example file is explicitly allowed — it's documentation, not a secret."""
error = get_read_block_error("/tmp/project/.env.example")
assert error is None
def test_allowed_env_sample(self):
"""Other .env variants like .env.sample are allowed."""
error = get_read_block_error("/tmp/project/.env.sample")
assert error is None
def test_allowed_non_env_files(self):
"""Regular files are not affected by the env guard."""
for path in ["/tmp/project/config.yaml", "/tmp/project/main.py",
"/tmp/project/README.md", "/tmp/project/.gitignore"]:
error = get_read_block_error(path)
assert error is None, f"{path} should be allowed"
def test_allowed_hermes_env(self):
"""Hermes' own .env inside HERMES_HOME is NOT blocked by this rule
(it's handled by other mechanisms). Only project-local .env is blocked."""
# Note: hermes internal .env is in ~/.hermes/.env which is NOT a project-local
# path, but the basename check applies to ANY .env. This is intentional —
# even ~/.hermes/.env should not be readable via read_file.
error = get_read_block_error(os.path.expanduser("~/.hermes/.env"))
assert error is not None
def test_blocked_set_is_lowercase(self):
"""All entries in the blocked set are lowercase for case-insensitive matching."""
for name in _BLOCKED_PROJECT_ENV_BASENAMES:
assert name == name.lower(), f"{name} should be lowercase"
# ---------------------------------------------------------------------------
# Existing cache-file blocking (regression — must still work)
# ---------------------------------------------------------------------------
class TestCacheFileReadBlocking:
"""Internal Hermes cache files must remain blocked."""
def test_hub_index_cache_blocked(self, tmp_path):
"""Hub index-cache reads are blocked."""
hermes_home = tmp_path / ".hermes"
cache = hermes_home / "skills" / ".hub" / "index-cache" / "data.json"
cache.parent.mkdir(parents=True)
cache.write_text("{}")
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
error = get_read_block_error(str(cache))
assert error is not None
assert "internal Hermes cache" in error
def test_hub_directory_blocked(self, tmp_path):
"""Hub directory reads are blocked."""
hermes_home = tmp_path / ".hermes"
hub = hermes_home / "skills" / ".hub" / "metadata.json"
hub.parent.mkdir(parents=True)
hub.write_text("{}")
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
error = get_read_block_error(str(hub))
assert error is not None
# ---------------------------------------------------------------------------
# Combined: env guard + cache guard don't interfere
# ---------------------------------------------------------------------------
class TestCombinedGuards:
"""Both guards should work independently without interference."""
def test_env_guard_works_regardless_of_hermes_home(self, tmp_path):
"""The env basename guard does not depend on HERMES_HOME resolution."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
# Regular project .env should still be blocked
error = get_read_block_error("/workspace/.env")
assert error is not None
# .env.example should still be allowed
error = get_read_block_error("/workspace/.env.example")
assert error is None
def test_cache_guard_still_works_with_env_guard(self, tmp_path):
"""Cache file blocking still works when env guard is active."""
hermes_home = tmp_path / ".hermes"
cache = hermes_home / "skills" / ".hub" / "index-cache" / "x"
cache.parent.mkdir(parents=True)
cache.write_text("")
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
error = get_read_block_error(str(cache))
assert error is not None
assert "internal Hermes cache" in error
+339
View File
@@ -0,0 +1,339 @@
"""Tests for HERMES_HOME credential-file read blocking in file_safety.
Regression for https://github.com/NousResearch/hermes-agent/issues/17656 —
``read_file`` was previously only sandboxed against ``HERMES_HOME`` itself,
which left ``auth.json`` and ``.anthropic_oauth.json`` (plaintext provider
keys + OAuth tokens) readable by the agent. A prompt-injection reaching
``read_file`` could exfiltrate active credentials.
These tests verify that ``get_read_block_error`` returns a denial message
for the credential stores while leaving arbitrary ``HERMES_HOME`` files
readable, and that the existing ``skills/.hub`` deny still applies.
"""
from __future__ import annotations
import os
from pathlib import Path
import pytest
@pytest.fixture()
def fake_home(tmp_path, monkeypatch):
"""Point ``_hermes_home_path()`` at a tmp dir for isolated checks."""
import agent.file_safety as fs
home = tmp_path / "hermes_home"
home.mkdir()
monkeypatch.setattr(fs, "_hermes_home_path", lambda: home)
return home
def _create(home: Path, rel: str | Path) -> Path:
"""Create the file (with parents) so realpath() resolves it."""
p = home / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text("dummy", encoding="utf-8")
return p
def test_auth_json_blocked(fake_home):
from agent.file_safety import get_read_block_error
auth = _create(fake_home, "auth.json")
err = get_read_block_error(str(auth))
assert err is not None
assert "credential store" in err
assert "auth.json" in err
def test_auth_lock_blocked(fake_home):
from agent.file_safety import get_read_block_error
lock = _create(fake_home, "auth.lock")
err = get_read_block_error(str(lock))
assert err is not None
assert "credential store" in err
def test_anthropic_oauth_json_blocked(fake_home):
from agent.file_safety import get_read_block_error
oauth = _create(fake_home, ".anthropic_oauth.json")
err = get_read_block_error(str(oauth))
assert err is not None
assert "credential store" in err
def test_google_oauth_json_blocked(fake_home):
"""Gemini OAuth tokens live under auth/google_oauth.json — blocked."""
from agent.file_safety import get_read_block_error
oauth = _create(fake_home, Path("auth") / "google_oauth.json")
err = get_read_block_error(str(oauth))
assert err is not None
assert "credential store" in err
def test_arbitrary_hermes_home_file_not_blocked(fake_home):
"""Non-credential files inside HERMES_HOME stay readable."""
from agent.file_safety import get_read_block_error
safe = _create(fake_home, "session_log.txt")
assert get_read_block_error(str(safe)) is None
def test_subdirectory_named_auth_json_not_blocked(fake_home):
"""Only the top-level auth.json is the credential store; a file with the
same name in a subdirectory (e.g., a skill mock) must remain readable."""
from agent.file_safety import get_read_block_error
nested = _create(fake_home, Path("skills") / "my-skill" / "auth.json")
assert get_read_block_error(str(nested)) is None
def test_skills_hub_block_still_applies(fake_home):
"""Regression guard: the original skills/.hub deny must keep working."""
from agent.file_safety import get_read_block_error
hub_file = _create(fake_home, "skills/.hub/manifest.json")
err = get_read_block_error(str(hub_file))
assert err is not None
assert "internal Hermes cache file" in err
def test_path_traversal_resolves_to_blocked(fake_home, tmp_path):
"""A path that traverses through a sibling dir back into HERMES_HOME's
auth.json must still be caught — the check resolves through realpath."""
from agent.file_safety import get_read_block_error
_create(fake_home, "auth.json")
sibling = tmp_path / "elsewhere"
sibling.mkdir()
traversal = sibling / ".." / "hermes_home" / "auth.json"
err = get_read_block_error(str(traversal))
assert err is not None
assert "credential store" in err
def test_symlink_to_auth_json_blocked(fake_home, tmp_path):
"""A symlink pointing at HERMES_HOME/auth.json from outside the home
must be blocked — readlink-resolution catches the indirection."""
from agent.file_safety import get_read_block_error
target = _create(fake_home, "auth.json")
link = tmp_path / "shim.json"
try:
os.symlink(target, link)
except (OSError, NotImplementedError):
pytest.skip("symlinks not supported on this platform/filesystem")
err = get_read_block_error(str(link))
assert err is not None
assert "credential store" in err
def test_read_file_tool_blocks_relative_path_under_terminal_cwd(
fake_home, tmp_path, monkeypatch
):
"""Bypass guard: a relative path like ``"auth.json"`` resolved by
``read_file_tool`` against ``TERMINAL_CWD == HERMES_HOME`` must still
be blocked, even though ``get_read_block_error``'s own ``resolve()``
is anchored at the (different) Python process cwd.
"""
import json
import tools.file_tools as ft
_create(fake_home, "auth.json")
# Force the file_tools resolver to anchor relative paths at HERMES_HOME
# while the Python process cwd remains tmp_path (a different directory).
monkeypatch.setenv("TERMINAL_CWD", str(fake_home))
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
ft, "_get_live_tracking_cwd", lambda task_id="default": None
)
out = json.loads(ft.read_file_tool("auth.json"))
assert "error" in out
assert "credential store" in out["error"]
def test_read_file_tool_blocks_nested_google_oauth_path(
fake_home, tmp_path, monkeypatch
):
"""The real read_file tool must not return Gemini OAuth token material."""
import json
import tools.file_tools as ft
oauth = _create(fake_home, Path("auth") / "google_oauth.json")
oauth.write_text(
json.dumps(
{
"refresh": "REFRESH_TOKEN_MARKER",
"access": "ACCESS_TOKEN_MARKER",
"email": "user@example.com",
}
),
encoding="utf-8",
)
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
ft, "_get_live_tracking_cwd", lambda task_id="default": None
)
out = json.loads(ft.read_file_tool(str(oauth), task_id="google-oauth-test"))
assert "error" in out
assert "credential store" in out["error"]
assert "REFRESH_TOKEN_MARKER" not in json.dumps(out)
assert "ACCESS_TOKEN_MARKER" not in json.dumps(out)
# ---------------------------------------------------------------------------
# Widening: .env, webhook_subscriptions.json, mcp-tokens/
# ---------------------------------------------------------------------------
def test_dotenv_blocked(fake_home):
""".env in HERMES_HOME holds API keys — blocked."""
from agent.file_safety import get_read_block_error
env = _create(fake_home, ".env")
err = get_read_block_error(str(env))
assert err is not None
assert "credential store" in err
def test_webhook_subscriptions_blocked(fake_home):
"""webhook_subscriptions.json holds per-route HMAC secrets — blocked."""
from agent.file_safety import get_read_block_error
subs = _create(fake_home, "webhook_subscriptions.json")
err = get_read_block_error(str(subs))
assert err is not None
assert "credential store" in err
def test_mcp_tokens_file_blocked(fake_home):
"""Files under mcp-tokens/ hold OAuth tokens — blocked."""
from agent.file_safety import get_read_block_error
tok = _create(fake_home, Path("mcp-tokens") / "github.json")
err = get_read_block_error(str(tok))
assert err is not None
assert "MCP token" in err
def test_mcp_tokens_nested_blocked(fake_home):
"""Nested files inside mcp-tokens/ are also blocked."""
from agent.file_safety import get_read_block_error
tok = _create(fake_home, Path("mcp-tokens") / "providers" / "azure.json")
err = get_read_block_error(str(tok))
assert err is not None
assert "MCP token" in err
def test_mcp_tokens_dir_itself_blocked(fake_home):
"""The mcp-tokens directory itself is blocked (listing is exfiltrating)."""
from agent.file_safety import get_read_block_error
tokens_dir = fake_home / "mcp-tokens"
tokens_dir.mkdir(parents=True, exist_ok=True)
err = get_read_block_error(str(tokens_dir))
assert err is not None
assert "MCP token" in err
def test_identically_named_hermes_files_outside_home_not_blocked(
fake_home, tmp_path
):
"""Hermes-specific filenames (``auth.json``, ``mcp-tokens/``, ``google_oauth.json``)
outside HERMES_HOME must remain readable — the gate is per-location for
those, not per-filename. ``.env`` is the exception: it's blocked anywhere
on disk (see test_project_local_env_blocked) because the basename always
means \"secret-bearing environment file\" regardless of directory."""
from agent.file_safety import get_read_block_error
project = tmp_path / "myproject"
project.mkdir()
# auth.json outside HERMES_HOME — readable (per-location gate).
p = project / "auth.json"
p.write_text("not secret here", encoding="utf-8")
assert get_read_block_error(str(p)) is None, (
"auth.json outside HERMES_HOME should NOT be blocked"
)
google_oauth = project / "auth" / "google_oauth.json"
google_oauth.parent.mkdir()
google_oauth.write_text("not really a token", encoding="utf-8")
assert get_read_block_error(str(google_oauth)) is None
tokens = project / "mcp-tokens"
tokens.mkdir()
tok_file = tokens / "token.json"
tok_file.write_text("not really a token", encoding="utf-8")
assert get_read_block_error(str(tok_file)) is None
def test_non_secret_auth_subtree_file_not_blocked(fake_home):
"""Only the known Google OAuth token path is blocked, not all auth/*."""
from agent.file_safety import get_read_block_error
note = _create(fake_home, Path("auth") / "notes.json")
assert get_read_block_error(str(note)) is None
def test_config_yaml_not_blocked(fake_home):
"""config.yaml is NOT a credential file — agent should still be
able to read it for debugging. (Writes are denied separately by
is_write_denied; reads stay allowed.)"""
from agent.file_safety import get_read_block_error
cfg = _create(fake_home, "config.yaml")
assert get_read_block_error(str(cfg)) is None
def test_profile_mode_blocks_root_credentials(tmp_path, monkeypatch):
"""Under a profile, HERMES_HOME = <root>/profiles/<name>, but
<root>/auth.json must ALSO be blocked — credentials at root are
inherited by every profile."""
import agent.file_safety as fs
root = tmp_path / "hermes"
profile = root / "profiles" / "coder"
profile.mkdir(parents=True)
monkeypatch.setattr(fs, "_hermes_home_path", lambda: profile)
monkeypatch.setattr(fs, "_hermes_root_path", lambda: root)
from agent.file_safety import get_read_block_error
# Profile-local credential store: blocked
profile_auth = profile / "auth.json"
profile_auth.write_text("x")
assert "credential store" in (get_read_block_error(str(profile_auth)) or "")
# Root-level credential store: ALSO blocked (this is the widening)
root_auth = root / "auth.json"
root_auth.write_text("x")
assert "credential store" in (get_read_block_error(str(root_auth)) or "")
# Root-level .env: blocked too
root_env = root / ".env"
root_env.write_text("x")
assert "credential store" in (get_read_block_error(str(root_env)) or "")
# Root-level Google OAuth token store: blocked too
root_google_oauth = root / "auth" / "google_oauth.json"
root_google_oauth.parent.mkdir(parents=True, exist_ok=True)
root_google_oauth.write_text("x")
assert "credential store" in (
get_read_block_error(str(root_google_oauth)) or ""
)
# Root-level mcp-tokens: blocked
root_tok = root / "mcp-tokens" / "gh.json"
root_tok.parent.mkdir(parents=True, exist_ok=True)
root_tok.write_text("x")
assert "MCP token" in (get_read_block_error(str(root_tok)) or "")
@@ -0,0 +1,218 @@
"""Tests for the cross-Hermes-profile write guard in agent/file_safety.
The guard fires when a tool tries to write into another Hermes profile's
skills/plugins/cron/memories directory. It's a soft guard — defense in
depth, NOT a security boundary — but it prevents the agent from silently
corrupting a profile that belongs to a different session.
Reference: May 2026 incident — a hermes-security profile session
accidentally edited skills under both ~/.hermes/profiles/hermes-security/skills/
AND ~/.hermes/skills/ (the default profile's skills), realizing only
afterwards that the second path belonged to a different profile.
"""
from __future__ import annotations
from pathlib import Path
import pytest
# ---------------------------------------------------------------------------
# Helpers — set up a fake Hermes root with two profiles, monkeypatch the
# resolver helpers so the classifier sees the test layout.
# ---------------------------------------------------------------------------
@pytest.fixture
def fake_hermes(tmp_path, monkeypatch):
"""Build a fake Hermes layout:
<tmp>/
skills/foo/SKILL.md # default profile
plugins/foo/__init__.py
cron/<state>
memories/MEMORY.md
profiles/
hermes-security/
skills/foo/SKILL.md # named profile
plugins/...
coder/
skills/foo/SKILL.md # another named profile
"""
root = tmp_path / "fake-hermes"
(root / "skills" / "foo").mkdir(parents=True)
(root / "skills" / "foo" / "SKILL.md").write_text("# default skill\n")
(root / "plugins" / "foo").mkdir(parents=True)
(root / "memories").mkdir(parents=True)
(root / "cron").mkdir(parents=True)
sec_home = root / "profiles" / "hermes-security"
(sec_home / "skills" / "foo").mkdir(parents=True)
(sec_home / "skills" / "foo" / "SKILL.md").write_text("# sec skill\n")
(sec_home / "plugins").mkdir(parents=True)
coder_home = root / "profiles" / "coder"
(coder_home / "skills" / "foo").mkdir(parents=True)
(coder_home / "skills" / "foo" / "SKILL.md").write_text("# coder skill\n")
# Monkeypatch the resolver functions used by file_safety so each test
# can choose which profile is "active".
import hermes_constants
monkeypatch.setattr(hermes_constants, "get_default_hermes_root", lambda: root)
# The reloads below ensure get_cross_profile_warning/classify see the patched root.
import agent.file_safety as fs
monkeypatch.setattr(fs, "_hermes_root_path", lambda: root)
return {
"root": root,
"default_home": root,
"security_home": sec_home,
"coder_home": coder_home,
}
def _set_active_home(monkeypatch, hermes_home: Path):
"""Point file_safety._hermes_home_path at a specific profile dir."""
import agent.file_safety as fs
monkeypatch.setattr(fs, "_hermes_home_path", lambda: hermes_home)
# ---------------------------------------------------------------------------
# _resolve_active_profile_name
# ---------------------------------------------------------------------------
class TestResolveActiveProfileName:
def test_default_when_home_is_root(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["default_home"])
from agent.file_safety import _resolve_active_profile_name
assert _resolve_active_profile_name() == "default"
def test_named_profile(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import _resolve_active_profile_name
assert _resolve_active_profile_name() == "hermes-security"
def test_falls_back_to_default_on_resolution_failure(self, fake_hermes, monkeypatch):
"""If HERMES_HOME resolution raises, return 'default' rather than crashing the tool."""
import agent.file_safety as fs
def _boom():
raise RuntimeError("simulated")
monkeypatch.setattr(fs, "_hermes_home_path", _boom)
# Should not raise — falls back to "default"
assert fs._resolve_active_profile_name() == "default"
# ---------------------------------------------------------------------------
# classify_cross_profile_target
# ---------------------------------------------------------------------------
class TestClassifyCrossProfileTarget:
def test_same_profile_write_returns_none(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is None
def test_security_writing_default_skill(self, fake_hermes, monkeypatch):
"""The exact incident from May 2026."""
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is not None
assert result["active_profile"] == "hermes-security"
assert result["target_profile"] == "default"
assert result["area"] == "skills"
def test_default_writing_security_skill(self, fake_hermes, monkeypatch):
"""Inverse direction — default-profile session reaching into a named profile."""
_set_active_home(monkeypatch, fake_hermes["default_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is not None
assert result["active_profile"] == "default"
assert result["target_profile"] == "hermes-security"
def test_named_to_named_cross_profile(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["coder_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is not None
assert result["target_profile"] == "coder"
@pytest.mark.parametrize("area", ["skills", "plugins", "cron", "memories"])
def test_all_profile_scoped_areas_classified(self, fake_hermes, monkeypatch, area):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
target = fake_hermes["default_home"] / area / "foo.txt"
result = classify_cross_profile_target(str(target))
assert result is not None
assert result["area"] == area
def test_non_hermes_path_returns_none(self, fake_hermes, monkeypatch, tmp_path):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
# Path outside any Hermes root
assert classify_cross_profile_target(str(tmp_path / "random.txt")) is None
def test_hermes_config_not_classified_as_cross_profile(self, fake_hermes, monkeypatch):
"""Files under <root>/config.yaml or <root>/.env are NOT profile-scoped
(already covered by build_write_denied_paths). Don't double-warn."""
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
# config.yaml at root level is not in PROFILE_SCOPED_AREAS
result = classify_cross_profile_target(
str(fake_hermes["default_home"] / "config.yaml")
)
assert result is None
# ---------------------------------------------------------------------------
# get_cross_profile_warning
# ---------------------------------------------------------------------------
class TestGetCrossProfileWarning:
def test_in_profile_returns_none(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import get_cross_profile_warning
assert get_cross_profile_warning(
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
) is None
def test_cross_profile_warning_names_both_profiles(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import get_cross_profile_warning
warn = get_cross_profile_warning(
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
)
assert warn is not None
# Must name BOTH profiles so the model knows which is which.
assert "default" in warn
assert "hermes-security" in warn
# Must name the bypass kwarg.
assert "cross_profile=True" in warn
# Must reference the area.
assert "skills" in warn
def test_warning_is_defense_in_depth_not_boundary(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import get_cross_profile_warning
warn = get_cross_profile_warning(
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
)
# Must self-document as defense-in-depth so future reviewers
# don't promote it to a hard block.
assert "not a security boundary" in warn.lower()
File diff suppressed because it is too large Load Diff
+78
View File
@@ -0,0 +1,78 @@
"""Regression tests for #13636 — CloudCode / Gemini CLI rate-limit fallback.
_pool_may_recover_from_rate_limit() is the hinge between credential-pool
rotation and fallback-provider activation. For CloudCode (Gemini CLI /
Gemini OAuth) the 429 is an account-wide throttle, so waiting for pool
rotation is pointless — prefer fallback immediately.
"""
import inspect
from unittest.mock import MagicMock
from agent import conversation_loop
from run_agent import _pool_may_recover_from_rate_limit
def _pool(entries: int = 2):
p = MagicMock()
p.has_available.return_value = True
p.entries.return_value = list(range(entries))
return p
def test_cloudcode_provider_skips_pool_rotation():
assert _pool_may_recover_from_rate_limit(
_pool(entries=3),
provider="google-gemini-cli",
base_url="cloudcode-pa://google",
) is False
def test_cloudcode_base_url_skips_pool_rotation_even_on_alias_provider():
# Even if the provider label is something else, a cloudcode-pa:// URL
# signals the account-wide quota regime.
assert _pool_may_recover_from_rate_limit(
_pool(entries=3),
provider="custom-provider",
base_url="cloudcode-pa://google",
) is False
def test_non_cloudcode_multi_entry_pool_still_recovers():
assert _pool_may_recover_from_rate_limit(
_pool(entries=3),
provider="openrouter",
base_url="https://openrouter.ai/api/v1",
) is True
def test_single_entry_pool_skips_rotation_regardless_of_provider():
# Pre-existing single-entry-pool exception (#11314) still holds.
assert _pool_may_recover_from_rate_limit(
_pool(entries=1),
provider="openrouter",
base_url="https://openrouter.ai/api/v1",
) is False
def test_exhausted_pool_skips_rotation():
p = MagicMock()
p.has_available.return_value = False
assert _pool_may_recover_from_rate_limit(p) is False
def test_no_pool_skips_rotation():
assert _pool_may_recover_from_rate_limit(None) is False
def test_conversation_loop_resolves_pool_helper_through_run_agent_module():
"""Extracted conversation loop must honor tests/patches on run_agent.
conversation_loop intentionally lazy-loads run_agent via _ra(). If this
call site uses a bare imported helper, monkeypatching run_agent in tests (and
production wrappers that patch run_agent) will not propagate into the
extracted loop; older code also hit NameError in this branch.
"""
source = inspect.getsource(conversation_loop.run_conversation)
assert "_ra()._pool_may_recover_from_rate_limit(" in source
assert "pool_may_recover = _pool_may_recover_from_rate_limit(" not in source
+165
View File
@@ -0,0 +1,165 @@
"""Tests for Gemini free-tier detection and blocking."""
from __future__ import annotations
from unittest.mock import MagicMock, patch
from agent.gemini_native_adapter import (
gemini_http_error,
is_free_tier_quota_error,
probe_gemini_tier,
)
def _mock_response(status: int, headers: dict | None = None, text: str = "") -> MagicMock:
resp = MagicMock()
resp.status_code = status
resp.headers = headers or {}
resp.text = text
return resp
def _run_probe(resp: MagicMock) -> str:
with patch("agent.gemini_native_adapter.httpx.Client") as MC:
inst = MagicMock()
inst.post.return_value = resp
MC.return_value.__enter__.return_value = inst
return probe_gemini_tier("fake-key")
class TestProbeGeminiTier:
"""Verify the tier probe classifies keys correctly."""
def test_free_tier_via_rpd_header_flash(self):
# gemini-2.5-flash free tier: 250 RPD
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "250"}, "{}")
assert _run_probe(resp) == "free"
def test_free_tier_via_rpd_header_pro(self):
# gemini-2.5-pro free tier: 100 RPD
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "100"}, "{}")
assert _run_probe(resp) == "free"
def test_free_tier_via_rpd_header_flash_lite(self):
# flash-lite free tier: 1000 RPD (our upper bound)
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "1000"}, "{}")
assert _run_probe(resp) == "free"
def test_paid_tier_via_rpd_header(self):
# Tier 1 starts at 1500+ RPD
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "1500"}, "{}")
assert _run_probe(resp) == "paid"
def test_free_tier_via_429_body(self):
body = (
'{"error":{"code":429,"message":"Quota exceeded for metric: '
'generativelanguage.googleapis.com/generate_content_free_tier_requests, '
'limit: 20"}}'
)
resp = _mock_response(429, {}, body)
assert _run_probe(resp) == "free"
def test_paid_429_has_no_free_tier_marker(self):
body = '{"error":{"code":429,"message":"rate limited"}}'
resp = _mock_response(429, {}, body)
assert _run_probe(resp) == "paid"
def test_successful_200_without_rpd_header_is_paid(self):
resp = _mock_response(200, {}, '{"candidates":[]}')
assert _run_probe(resp) == "paid"
def test_401_returns_unknown(self):
resp = _mock_response(401, {}, '{"error":{"code":401}}')
assert _run_probe(resp) == "unknown"
def test_404_returns_unknown(self):
resp = _mock_response(404, {}, '{"error":{"code":404}}')
assert _run_probe(resp) == "unknown"
def test_network_error_returns_unknown(self):
with patch(
"agent.gemini_native_adapter.httpx.Client",
side_effect=Exception("dns failure"),
):
assert probe_gemini_tier("fake-key") == "unknown"
def test_empty_key_returns_unknown(self):
assert probe_gemini_tier("") == "unknown"
assert probe_gemini_tier(" ") == "unknown"
assert probe_gemini_tier(None) == "unknown" # type: ignore[arg-type]
def test_malformed_rpd_header_falls_through(self):
# Non-integer header value shouldn't crash; 200 with no usable header -> paid.
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "abc"}, "{}")
assert _run_probe(resp) == "paid"
def test_openai_compat_suffix_stripped(self):
"""Base URLs ending in /openai get normalized to the native endpoint."""
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "1500"}, "{}")
with patch("agent.gemini_native_adapter.httpx.Client") as MC:
inst = MagicMock()
inst.post.return_value = resp
MC.return_value.__enter__.return_value = inst
probe_gemini_tier(
"fake",
"https://generativelanguage.googleapis.com/v1beta/openai",
)
# Verify the post URL does NOT contain /openai
called_url = inst.post.call_args[0][0]
assert "/openai/" not in called_url
assert called_url.endswith(":generateContent")
class TestIsFreeTierQuotaError:
def test_detects_free_tier_marker(self):
assert is_free_tier_quota_error(
"Quota exceeded for metric: generate_content_free_tier_requests"
)
def test_case_insensitive(self):
assert is_free_tier_quota_error("QUOTA: FREE_TIER_REQUESTS")
def test_no_free_tier_marker(self):
assert not is_free_tier_quota_error("rate limited")
def test_empty_string(self):
assert not is_free_tier_quota_error("")
def test_none(self):
assert not is_free_tier_quota_error(None) # type: ignore[arg-type]
class TestGeminiHttpErrorFreeTierGuidance:
"""gemini_http_error should append free-tier guidance for free-tier 429s."""
class _FakeResp:
def __init__(self, status: int, text: str):
self.status_code = status
self.headers: dict = {}
self.text = text
def test_free_tier_429_appends_guidance(self):
body = (
'{"error":{"code":429,"message":"Quota exceeded for metric: '
"generativelanguage.googleapis.com/generate_content_free_tier_requests, "
'limit: 20","status":"RESOURCE_EXHAUSTED"}}'
)
err = gemini_http_error(self._FakeResp(429, body))
msg = str(err)
assert "free tier" in msg.lower()
assert "aistudio.google.com/apikey" in msg
def test_paid_429_has_no_billing_url(self):
body = '{"error":{"code":429,"message":"Rate limited","status":"RESOURCE_EXHAUSTED"}}'
err = gemini_http_error(self._FakeResp(429, body))
assert "aistudio.google.com/apikey" not in str(err)
def test_non_429_has_no_billing_url(self):
body = '{"error":{"code":400,"message":"bad request","status":"INVALID_ARGUMENT"}}'
err = gemini_http_error(self._FakeResp(400, body))
assert "aistudio.google.com/apikey" not in str(err)
def test_401_has_no_billing_url(self):
body = '{"error":{"code":401,"message":"API key invalid","status":"UNAUTHENTICATED"}}'
err = gemini_http_error(self._FakeResp(401, body))
assert "aistudio.google.com/apikey" not in str(err)
+328
View File
@@ -0,0 +1,328 @@
"""Tests for the native Google AI Studio Gemini adapter."""
from __future__ import annotations
import json
from types import SimpleNamespace
import pytest
class DummyResponse:
def __init__(self, status_code=200, payload=None, headers=None, text=None):
self.status_code = status_code
self._payload = payload or {}
self.headers = headers or {}
self.text = text if text is not None else json.dumps(self._payload)
def json(self):
return self._payload
def test_build_native_request_preserves_thought_signature_on_tool_replay():
from agent.gemini_native_adapter import build_gemini_request
request = build_gemini_request(
messages=[
{"role": "system", "content": "Be helpful."},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
"extra_content": {
"google": {"thought_signature": "sig-123"}
},
}
],
},
],
tools=[],
tool_choice=None,
)
parts = request["contents"][0]["parts"]
assert parts[0]["functionCall"]["name"] == "get_weather"
assert parts[0]["thoughtSignature"] == "sig-123"
def test_build_native_request_uses_original_function_name_for_tool_result():
from agent.gemini_native_adapter import build_gemini_request
request = build_gemini_request(
messages=[
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": '{"forecast": "sunny"}',
},
],
tools=[],
tool_choice=None,
)
tool_response = request["contents"][1]["parts"][0]["functionResponse"]
assert tool_response["name"] == "get_weather"
def test_build_native_request_strips_json_schema_only_fields_from_tool_parameters():
from agent.gemini_native_adapter import build_gemini_request
request = build_gemini_request(
messages=[{"role": "user", "content": "Hello"}],
tools=[
{
"type": "function",
"function": {
"name": "lookup_weather",
"description": "Weather lookup",
"parameters": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"additionalProperties": False,
"properties": {
"city": {
"type": "string",
"$schema": "ignored",
"description": "City name",
}
},
"required": ["city"],
},
},
}
],
tool_choice=None,
)
params = request["tools"][0]["functionDeclarations"][0]["parameters"]
assert "$schema" not in params
assert "additionalProperties" not in params
assert params["type"] == "object"
assert params["properties"]["city"] == {
"type": "string",
"description": "City name",
}
def test_translate_native_response_surfaces_reasoning_and_tool_calls():
from agent.gemini_native_adapter import translate_gemini_response
payload = {
"candidates": [
{
"content": {
"parts": [
{"thought": True, "text": "thinking..."},
{"functionCall": {"name": "search", "args": {"q": "hermes"}}},
]
},
"finishReason": "STOP",
}
],
"usageMetadata": {
"promptTokenCount": 10,
"candidatesTokenCount": 5,
"totalTokenCount": 15,
},
}
response = translate_gemini_response(payload, model="gemini-2.5-flash")
choice = response.choices[0]
assert choice.finish_reason == "tool_calls"
assert choice.message.reasoning == "thinking..."
assert choice.message.tool_calls[0].function.name == "search"
assert json.loads(choice.message.tool_calls[0].function.arguments) == {"q": "hermes"}
def test_native_client_uses_x_goog_api_key_and_native_models_endpoint(monkeypatch):
from agent.gemini_native_adapter import GeminiNativeClient
recorded = {}
class DummyHTTP:
def post(self, url, json=None, headers=None, timeout=None):
recorded["url"] = url
recorded["json"] = json
recorded["headers"] = headers
return DummyResponse(
payload={
"candidates": [
{
"content": {"parts": [{"text": "hello"}]},
"finishReason": "STOP",
}
],
"usageMetadata": {
"promptTokenCount": 1,
"candidatesTokenCount": 1,
"totalTokenCount": 2,
},
}
)
def close(self):
return None
monkeypatch.setattr("agent.gemini_native_adapter.httpx.Client", lambda *a, **k: DummyHTTP())
client = GeminiNativeClient(api_key="AIza-test", base_url="https://generativelanguage.googleapis.com/v1beta")
response = client.chat.completions.create(
model="gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello"}],
)
assert recorded["url"] == "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent"
assert recorded["headers"]["x-goog-api-key"] == "AIza-test"
assert "Authorization" not in recorded["headers"]
assert response.choices[0].message.content == "hello"
def test_native_http_error_keeps_status_and_retry_after():
from agent.gemini_native_adapter import gemini_http_error
response = DummyResponse(
status_code=429,
headers={"Retry-After": "17"},
payload={
"error": {
"code": 429,
"message": "quota exhausted",
"status": "RESOURCE_EXHAUSTED",
"details": [
{
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
"reason": "RESOURCE_EXHAUSTED",
"metadata": {"service": "generativelanguage.googleapis.com"},
}
],
}
},
)
err = gemini_http_error(response)
assert getattr(err, "status_code", None) == 429
assert getattr(err, "retry_after", None) == 17.0
assert "quota exhausted" in str(err)
def test_native_client_accepts_injected_http_client():
from agent.gemini_native_adapter import GeminiNativeClient
injected = SimpleNamespace(close=lambda: None)
client = GeminiNativeClient(api_key="AIza-test", http_client=injected)
assert client._http is injected
def test_native_client_rejects_empty_api_key_with_actionable_message():
"""Empty/whitespace api_key must raise at construction, not produce a cryptic
Google GFE 'Error 400 (Bad Request)!!1' HTML page on the first request."""
from agent.gemini_native_adapter import GeminiNativeClient
for bad in ("", " ", None):
with pytest.raises(RuntimeError) as excinfo:
GeminiNativeClient(api_key=bad) # type: ignore[arg-type]
msg = str(excinfo.value)
assert "GOOGLE_API_KEY" in msg and "GEMINI_API_KEY" in msg
assert "aistudio.google.com" in msg
@pytest.mark.asyncio
async def test_async_native_client_streams_without_requiring_async_iterator_from_sync_client():
from agent.gemini_native_adapter import AsyncGeminiNativeClient
chunk = SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content="hi"), finish_reason=None)])
sync_stream = iter([chunk])
def _advance(iterator):
try:
return False, next(iterator)
except StopIteration:
return True, None
sync_client = SimpleNamespace(
api_key="AIza-test",
base_url="https://generativelanguage.googleapis.com/v1beta",
chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **kwargs: sync_stream)),
_advance_stream_iterator=_advance,
close=lambda: None,
)
async_client = AsyncGeminiNativeClient(sync_client)
stream = await async_client.chat.completions.create(stream=True)
collected = []
async for item in stream:
collected.append(item)
assert collected == [chunk]
def test_stream_event_translation_emits_tool_call_delta_with_stable_index():
from agent.gemini_native_adapter import translate_stream_event
tool_call_indices = {}
event = {
"candidates": [
{
"content": {
"parts": [
{"functionCall": {"name": "search", "args": {"q": "abc"}}}
]
},
"finishReason": "STOP",
}
]
}
first = translate_stream_event(event, model="gemini-2.5-flash", tool_call_indices=tool_call_indices)
second = translate_stream_event(event, model="gemini-2.5-flash", tool_call_indices=tool_call_indices)
assert first[0].choices[0].delta.tool_calls[0].index == 0
assert second[0].choices[0].delta.tool_calls[0].index == 0
assert first[0].choices[0].delta.tool_calls[0].id == second[0].choices[0].delta.tool_calls[0].id
assert first[0].choices[0].delta.tool_calls[0].function.arguments == '{"q": "abc"}'
assert second[0].choices[0].delta.tool_calls[0].function.arguments == ""
assert first[-1].choices[0].finish_reason == "tool_calls"
def test_stream_event_translation_keeps_identical_calls_in_distinct_parts():
from agent.gemini_native_adapter import translate_stream_event
event = {
"candidates": [
{
"content": {
"parts": [
{"functionCall": {"name": "search", "args": {"q": "abc"}}},
{"functionCall": {"name": "search", "args": {"q": "abc"}}},
]
},
"finishReason": "STOP",
}
]
}
chunks = translate_stream_event(event, model="gemini-2.5-flash", tool_call_indices={})
tool_chunks = [chunk for chunk in chunks if chunk.choices[0].delta.tool_calls]
assert tool_chunks[0].choices[0].delta.tool_calls[0].index == 0
assert tool_chunks[1].choices[0].delta.tool_calls[0].index == 1
assert tool_chunks[0].choices[0].delta.tool_calls[0].id != tool_chunks[1].choices[0].delta.tool_calls[0].id
+140
View File
@@ -0,0 +1,140 @@
"""Tests for agent.gemini_schema — OpenAI→Gemini tool parameter translation."""
from agent.gemini_schema import (
sanitize_gemini_schema,
sanitize_gemini_tool_parameters,
)
class TestSanitizeGeminiSchema:
def test_strips_unknown_top_level_keys(self):
"""$schema / additionalProperties etc. must not reach Gemini."""
schema = {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": False,
"properties": {"foo": {"type": "string"}},
}
cleaned = sanitize_gemini_schema(schema)
assert "$schema" not in cleaned
assert "additionalProperties" not in cleaned
assert cleaned["type"] == "object"
assert cleaned["properties"] == {"foo": {"type": "string"}}
def test_preserves_string_enums(self):
"""String-valued enums are valid for Gemini and must pass through."""
schema = {"type": "string", "enum": ["pending", "done", "cancelled"]}
cleaned = sanitize_gemini_schema(schema)
assert cleaned["type"] == "string"
assert cleaned["enum"] == ["pending", "done", "cancelled"]
def test_drops_integer_enum_to_satisfy_gemini(self):
"""Gemini rejects int-typed enums; the sanitizer must drop the enum.
Regression for the Discord tool's ``auto_archive_duration``:
``{type: integer, enum: [60, 1440, 4320, 10080]}`` caused
Gemini HTTP 400 INVALID_ARGUMENT
"Invalid value ... (TYPE_STRING), 60" on every request that
shipped the full tool catalog to generativelanguage.googleapis.com.
"""
schema = {
"type": "integer",
"enum": [60, 1440, 4320, 10080],
"description": "Minutes (60, 1440, 4320, 10080).",
}
cleaned = sanitize_gemini_schema(schema)
assert cleaned["type"] == "integer"
assert "enum" not in cleaned
# description must survive so the model still sees the allowed values
assert cleaned["description"].startswith("Minutes")
def test_drops_number_enum(self):
"""Same rule applies to ``type: number``."""
schema = {"type": "number", "enum": [0.5, 1.0, 2.0]}
cleaned = sanitize_gemini_schema(schema)
assert cleaned["type"] == "number"
assert "enum" not in cleaned
def test_drops_boolean_enum(self):
"""And to ``type: boolean`` (Gemini rejects non-string entries)."""
schema = {"type": "boolean", "enum": [True, False]}
cleaned = sanitize_gemini_schema(schema)
assert cleaned["type"] == "boolean"
assert "enum" not in cleaned
def test_keeps_string_enum_even_when_numeric_values_coexist_as_strings(self):
"""Stringified-numeric enums ARE valid for Gemini; don't drop them."""
schema = {"type": "string", "enum": ["60", "1440", "4320", "10080"]}
cleaned = sanitize_gemini_schema(schema)
assert cleaned["enum"] == ["60", "1440", "4320", "10080"]
def test_drops_nested_integer_enum_inside_properties(self):
"""The fix must apply recursively — the Discord case is nested."""
schema = {
"type": "object",
"properties": {
"auto_archive_duration": {
"type": "integer",
"enum": [60, 1440, 4320, 10080],
"description": "Thread archive duration in minutes.",
},
"status": {
"type": "string",
"enum": ["active", "archived"],
},
},
}
cleaned = sanitize_gemini_schema(schema)
props = cleaned["properties"]
# Integer enum is dropped...
assert props["auto_archive_duration"]["type"] == "integer"
assert "enum" not in props["auto_archive_duration"]
# ...but the sibling string enum is preserved.
assert props["status"]["enum"] == ["active", "archived"]
def test_drops_integer_enum_inside_array_items(self):
"""Array item schemas recurse through ``items``."""
schema = {
"type": "array",
"items": {"type": "integer", "enum": [1, 2, 3]},
}
cleaned = sanitize_gemini_schema(schema)
assert cleaned["items"]["type"] == "integer"
assert "enum" not in cleaned["items"]
def test_non_dict_input_returns_empty(self):
assert sanitize_gemini_schema(None) == {}
assert sanitize_gemini_schema("not a schema") == {}
assert sanitize_gemini_schema([1, 2, 3]) == {}
class TestSanitizeGeminiToolParameters:
def test_empty_parameters_return_valid_object_schema(self):
"""Gemini requires ``parameters`` to be a valid object schema."""
cleaned = sanitize_gemini_tool_parameters({})
assert cleaned == {"type": "object", "properties": {}}
def test_discord_create_thread_parameters_no_longer_trip_gemini(self):
"""End-to-end regression: the exact shape that was rejected in prod."""
params = {
"type": "object",
"properties": {
"action": {"type": "string", "enum": ["create_thread"]},
"auto_archive_duration": {
"type": "integer",
"enum": [60, 1440, 4320, 10080],
"description": "Thread archive duration in minutes "
"(create_thread, default 1440).",
},
},
"required": ["action"],
}
cleaned = sanitize_gemini_tool_parameters(params)
aad = cleaned["properties"]["auto_archive_duration"]
# The field that triggered the Gemini 400 is gone.
assert "enum" not in aad
# Type + description survive so the model still knows what to send.
assert aad["type"] == "integer"
assert "1440" in aad["description"]
# And the string-enum sibling is untouched.
assert cleaned["properties"]["action"]["enum"] == ["create_thread"]
+169
View File
@@ -0,0 +1,169 @@
"""Tests for agent.i18n -- catalog parity, fallback, language resolution."""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from agent import i18n
LOCALES_DIR = Path(__file__).resolve().parents[2] / "locales"
def _load_raw(lang: str) -> dict:
with (LOCALES_DIR / f"{lang}.yaml").open("r", encoding="utf-8") as f:
return yaml.safe_load(f)
def _flatten(d, prefix="") -> dict:
flat = {}
for k, v in (d or {}).items():
key = f"{prefix}.{k}" if prefix else k
if isinstance(v, dict):
flat.update(_flatten(v, key))
else:
flat[key] = v
return flat
# ---------------------------------------------------------------------------
# Catalog completeness -- this is the key invariant test. If someone adds a
# new key to en.yaml they MUST add it to every other locale, else runtime
# falls back to English for those users and defeats the feature.
# ---------------------------------------------------------------------------
def test_all_locales_exist():
"""Every supported language must have a catalog file on disk."""
for lang in i18n.SUPPORTED_LANGUAGES:
assert (LOCALES_DIR / f"{lang}.yaml").is_file(), f"missing locales/{lang}.yaml"
@pytest.mark.parametrize("lang", [l for l in i18n.SUPPORTED_LANGUAGES if l != "en"])
def test_catalog_keys_match_english(lang: str):
"""Every non-English catalog must have exactly the same key set as English."""
en_keys = set(_flatten(_load_raw("en")).keys())
lang_keys = set(_flatten(_load_raw(lang)).keys())
missing = en_keys - lang_keys
extra = lang_keys - en_keys
assert not missing, f"{lang}.yaml missing keys: {sorted(missing)}"
assert not extra, f"{lang}.yaml has keys not in en.yaml: {sorted(extra)}"
@pytest.mark.parametrize("lang", list(i18n.SUPPORTED_LANGUAGES))
def test_catalog_placeholders_match_english(lang: str):
"""Every translated value must use the same {placeholder} tokens as English.
A mistranslated placeholder (e.g. ``{description}`` typoed as ``{descricao}``)
would either raise KeyError at runtime or silently drop the interpolated
value. Pin parity at the test layer.
"""
import re
placeholder_re = re.compile(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}")
en_flat = _flatten(_load_raw("en"))
lang_flat = _flatten(_load_raw(lang))
for key, en_value in en_flat.items():
en_placeholders = set(placeholder_re.findall(en_value))
lang_value = lang_flat.get(key, "")
lang_placeholders = set(placeholder_re.findall(lang_value))
assert en_placeholders == lang_placeholders, (
f"{lang}.yaml key={key!r}: placeholders {lang_placeholders} "
f"don't match English {en_placeholders}"
)
# ---------------------------------------------------------------------------
# Language resolution
# ---------------------------------------------------------------------------
def test_normalize_lang_accepts_supported():
assert i18n._normalize_lang("zh") == "zh"
assert i18n._normalize_lang("EN") == "en"
def test_normalize_lang_accepts_aliases():
assert i18n._normalize_lang("chinese") == "zh"
assert i18n._normalize_lang("zh-CN") == "zh"
assert i18n._normalize_lang("Deutsch") == "de"
assert i18n._normalize_lang("español") == "es"
assert i18n._normalize_lang("jp") == "ja"
assert i18n._normalize_lang("Ukrainian") == "uk"
assert i18n._normalize_lang("uk-UA") == "uk"
assert i18n._normalize_lang("ua") == "uk"
assert i18n._normalize_lang("Turkish") == "tr"
assert i18n._normalize_lang("tr-TR") == "tr"
assert i18n._normalize_lang("türkçe") == "tr"
def test_normalize_lang_unknown_falls_back():
assert i18n._normalize_lang("klingon") == "en"
assert i18n._normalize_lang("") == "en"
assert i18n._normalize_lang(None) == "en"
def test_env_var_override(monkeypatch):
"""HERMES_LANGUAGE wins over config."""
i18n.reset_language_cache()
monkeypatch.setenv("HERMES_LANGUAGE", "ja")
assert i18n.get_language() == "ja"
def test_env_var_normalized(monkeypatch):
i18n.reset_language_cache()
monkeypatch.setenv("HERMES_LANGUAGE", "Chinese")
assert i18n.get_language() == "zh"
def test_default_when_nothing_set(monkeypatch):
"""With no env var and no config override, falls back to English."""
monkeypatch.delenv("HERMES_LANGUAGE", raising=False)
# Force config lookup to return None -- patch the cached reader.
i18n.reset_language_cache()
monkeypatch.setattr(i18n, "_config_language_cached", lambda: None)
assert i18n.get_language() == "en"
# ---------------------------------------------------------------------------
# t() semantics
# ---------------------------------------------------------------------------
def test_t_explicit_lang():
assert i18n.t("approval.denied", lang="en").endswith("Denied")
assert i18n.t("approval.denied", lang="zh").endswith("已拒绝")
assert i18n.t("approval.denied", lang="uk").endswith("Відхилено")
assert i18n.t("approval.denied", lang="tr").endswith("Reddedildi")
def test_t_formats_placeholders():
msg = i18n.t("gateway.draining", lang="en", count=3)
assert "3" in msg
def test_t_missing_key_returns_key():
"""A missing key returns its own path -- ugly but never crashes."""
result = i18n.t("nonexistent.key.path", lang="en")
assert result == "nonexistent.key.path"
def test_t_missing_key_in_non_english_falls_back_to_english(tmp_path, monkeypatch):
"""If a key exists in English but not in the target locale, fall back."""
# Stand up a fake incomplete locale under a temp locales dir.
fake_locales = tmp_path / "locales"
fake_locales.mkdir()
(fake_locales / "en.yaml").write_text("foo: English Foo\n", encoding="utf-8")
(fake_locales / "zh.yaml").write_text("# intentionally empty\n", encoding="utf-8")
monkeypatch.setattr(i18n, "_locales_dir", lambda: fake_locales)
i18n.reset_language_cache()
try:
assert i18n.t("foo", lang="zh") == "English Foo"
finally:
# Clear the cache on teardown so subsequent tests don't see the
# fake "foo: English Foo" catalog instead of the real locales/*.yaml.
i18n.reset_language_cache()
def test_t_unknown_language_uses_english():
"""Unknown lang codes normalize to English, not to a key-path fallback."""
assert i18n.t("approval.denied", lang="klingon") == i18n.t("approval.denied", lang="en")
+111
View File
@@ -0,0 +1,111 @@
"""Tests for agent/image_gen_registry.py — provider registration & active lookup."""
from __future__ import annotations
import pytest
from agent import image_gen_registry
from agent.image_gen_provider import ImageGenProvider
class _FakeProvider(ImageGenProvider):
def __init__(self, name: str, available: bool = True):
self._name = name
self._available = available
@property
def name(self) -> str:
return self._name
def is_available(self) -> bool:
return self._available
def generate(self, prompt, aspect_ratio="landscape", **kw):
return {"success": True, "image": f"{self._name}://{prompt}"}
@pytest.fixture(autouse=True)
def _reset_registry():
image_gen_registry._reset_for_tests()
yield
image_gen_registry._reset_for_tests()
class TestRegisterProvider:
def test_register_and_lookup(self):
provider = _FakeProvider("fake")
image_gen_registry.register_provider(provider)
assert image_gen_registry.get_provider("fake") is provider
def test_rejects_non_provider(self):
with pytest.raises(TypeError):
image_gen_registry.register_provider("not a provider") # type: ignore[arg-type]
def test_rejects_empty_name(self):
class Empty(ImageGenProvider):
@property
def name(self) -> str:
return ""
def generate(self, prompt, aspect_ratio="landscape", **kw):
return {}
with pytest.raises(ValueError):
image_gen_registry.register_provider(Empty())
def test_reregister_overwrites(self):
a = _FakeProvider("same")
b = _FakeProvider("same")
image_gen_registry.register_provider(a)
image_gen_registry.register_provider(b)
assert image_gen_registry.get_provider("same") is b
def test_list_is_sorted(self):
image_gen_registry.register_provider(_FakeProvider("zeta"))
image_gen_registry.register_provider(_FakeProvider("alpha"))
names = [p.name for p in image_gen_registry.list_providers()]
assert names == ["alpha", "zeta"]
class TestGetActiveProvider:
def test_single_provider_autoresolves(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
image_gen_registry.register_provider(_FakeProvider("solo"))
active = image_gen_registry.get_active_provider()
assert active is not None and active.name == "solo"
def test_fal_preferred_on_multi_without_config(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
image_gen_registry.register_provider(_FakeProvider("fal"))
image_gen_registry.register_provider(_FakeProvider("openai"))
active = image_gen_registry.get_active_provider()
assert active is not None and active.name == "fal"
def test_explicit_config_wins(self, tmp_path, monkeypatch):
import yaml
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"provider": "openai"}})
)
image_gen_registry.register_provider(_FakeProvider("fal"))
image_gen_registry.register_provider(_FakeProvider("openai"))
active = image_gen_registry.get_active_provider()
assert active is not None and active.name == "openai"
def test_missing_configured_provider_falls_back(self, tmp_path, monkeypatch):
import yaml
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"provider": "replicate"}})
)
# Only FAL is registered — configured provider doesn't exist
image_gen_registry.register_provider(_FakeProvider("fal"))
active = image_gen_registry.get_active_provider()
# Falls back to FAL preference (legacy default) rather than None
assert active is not None and active.name == "fal"
def test_none_when_empty(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
assert image_gen_registry.get_active_provider() is None
+638
View File
@@ -0,0 +1,638 @@
"""Tests for agent/image_routing.py — the per-turn image input mode decision."""
from __future__ import annotations
import base64
from pathlib import Path
from unittest.mock import patch
from agent.image_routing import (
_coerce_capability_bool,
_coerce_mode,
_explicit_aux_vision_override,
_lookup_supports_vision,
_supports_vision_override,
build_native_content_parts,
decide_image_input_mode,
extract_image_refs,
)
# ─── _coerce_mode ────────────────────────────────────────────────────────────
class TestCoerceMode:
def test_valid_modes_pass_through(self):
assert _coerce_mode("auto") == "auto"
assert _coerce_mode("native") == "native"
assert _coerce_mode("text") == "text"
def test_case_insensitive(self):
assert _coerce_mode("NATIVE") == "native"
assert _coerce_mode("Auto") == "auto"
def test_invalid_falls_back_to_auto(self):
assert _coerce_mode("nonsense") == "auto"
assert _coerce_mode("") == "auto"
assert _coerce_mode(None) == "auto"
assert _coerce_mode(42) == "auto"
def test_strips_whitespace(self):
assert _coerce_mode(" native ") == "native"
# ─── _explicit_aux_vision_override ───────────────────────────────────────────
class TestExplicitAuxVisionOverride:
def test_none_config(self):
assert _explicit_aux_vision_override(None) is False
def test_empty_config(self):
assert _explicit_aux_vision_override({}) is False
def test_default_auto_is_not_explicit(self):
cfg = {"auxiliary": {"vision": {"provider": "auto", "model": "", "base_url": ""}}}
assert _explicit_aux_vision_override(cfg) is False
def test_provider_set_is_explicit(self):
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": ""}}}
assert _explicit_aux_vision_override(cfg) is True
def test_model_set_is_explicit(self):
cfg = {"auxiliary": {"vision": {"provider": "auto", "model": "google/gemini-2.5-flash"}}}
assert _explicit_aux_vision_override(cfg) is True
def test_base_url_set_is_explicit(self):
cfg = {"auxiliary": {"vision": {"provider": "auto", "base_url": "http://localhost:11434"}}}
assert _explicit_aux_vision_override(cfg) is True
# ─── decide_image_input_mode ─────────────────────────────────────────────────
class TestDecideImageInputMode:
def test_explicit_native_overrides_everything(self):
cfg = {"agent": {"image_input_mode": "native"}}
# Non-vision model, aux-vision explicitly configured: native still wins.
cfg["auxiliary"] = {"vision": {"provider": "openrouter", "model": "foo"}}
with patch("agent.image_routing._lookup_supports_vision", return_value=False):
assert decide_image_input_mode("openrouter", "some-non-vision-model", cfg) == "native"
def test_explicit_text_overrides_everything(self):
cfg = {"agent": {"image_input_mode": "text"}}
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "text"
def test_auto_with_vision_capable_model(self):
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
assert decide_image_input_mode("anthropic", "claude-sonnet-4", {}) == "native"
def test_auto_with_non_vision_model(self):
with patch("agent.image_routing._lookup_supports_vision", return_value=False):
assert decide_image_input_mode("openrouter", "qwen/qwen3-235b", {}) == "text"
def test_auto_with_unknown_model(self):
with patch("agent.image_routing._lookup_supports_vision", return_value=None):
assert decide_image_input_mode("openrouter", "brand-new-slug", {}) == "text"
def test_auto_respects_aux_vision_override_even_for_vision_model(self):
"""If the user configured a dedicated vision backend, don't bypass it."""
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"}}}
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "text"
def test_none_config_is_auto(self):
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
assert decide_image_input_mode("anthropic", "claude-sonnet-4", None) == "native"
def test_invalid_mode_coerces_to_auto(self):
cfg = {"agent": {"image_input_mode": "weird-value"}}
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "native"
def test_auto_uses_text_for_text_only_modalities_even_with_attachment_flag(self):
registry = {
"xiaomi": {
"models": {
"mimo-v2.5-pro": {
"attachment": True,
"modalities": {"input": ["text"]},
"tool_call": True,
},
},
},
}
with patch("agent.models_dev.fetch_models_dev", return_value=registry):
assert decide_image_input_mode("xiaomi", "mimo-v2.5-pro", {}) == "text"
# ─── _coerce_capability_bool ─────────────────────────────────────────────────
class TestCoerceCapabilityBool:
def test_real_bool_passes_through(self):
assert _coerce_capability_bool(True) is True
assert _coerce_capability_bool(False) is False
def test_int_0_and_1(self):
assert _coerce_capability_bool(1) is True
assert _coerce_capability_bool(0) is False
def test_other_ints_return_none(self):
assert _coerce_capability_bool(2) is None
assert _coerce_capability_bool(-1) is None
def test_yaml_true_tokens(self):
for s in ("true", "TRUE", "True", "yes", "on", "1", " true "):
assert _coerce_capability_bool(s) is True
def test_yaml_false_tokens(self):
for s in ("false", "FALSE", "False", "no", "off", "0", " false "):
assert _coerce_capability_bool(s) is False
def test_quoted_false_does_not_silently_become_true(self):
# Regression: bool("false") is True in Python. A user writing
# supports_vision: "false" must NOT enable native vision routing.
assert _coerce_capability_bool("false") is False
def test_unrecognised_strings_return_none(self):
# None == fall through to models.dev, not a silent truthy.
assert _coerce_capability_bool("maybe") is None
assert _coerce_capability_bool("") is None
assert _coerce_capability_bool("definitely") is None
def test_other_types_return_none(self):
assert _coerce_capability_bool(None) is None
assert _coerce_capability_bool([]) is None
assert _coerce_capability_bool({}) is None
assert _coerce_capability_bool(1.5) is None
# ─── _supports_vision_override ───────────────────────────────────────────────
class TestSupportsVisionOverride:
def test_no_cfg_returns_none(self):
assert _supports_vision_override(None, "custom", "my-llava") is None
assert _supports_vision_override({}, "custom", "my-llava") is None
def test_top_level_shortcut_wins(self):
cfg = {"model": {"supports_vision": True}}
assert _supports_vision_override(cfg, "custom", "my-llava") is True
def test_top_level_false_propagates(self):
cfg = {"model": {"supports_vision": False}}
assert _supports_vision_override(cfg, "custom", "my-llava") is False
def test_per_provider_per_model_via_runtime_name(self):
cfg = {
"providers": {
"custom": {"models": {"my-llava": {"supports_vision": True}}},
},
}
assert _supports_vision_override(cfg, "custom", "my-llava") is True
def test_per_provider_per_model_via_config_name(self):
# Named custom provider — runtime self.provider == "custom", config
# holds the original name under model.provider.
cfg = {
"model": {"provider": "my-vllm"},
"providers": {
"my-vllm": {"models": {"my-llava": {"supports_vision": True}}},
},
}
assert _supports_vision_override(cfg, "custom", "my-llava") is True
def test_quoted_false_string_in_yaml_does_not_enable(self):
# Real-world: user writes supports_vision: "false" (quoted).
cfg = {"model": {"supports_vision": "false"}}
assert _supports_vision_override(cfg, "custom", "my-llava") is False
def test_unrecognised_value_falls_through(self):
cfg = {"model": {"supports_vision": "maybe"}}
assert _supports_vision_override(cfg, "custom", "my-llava") is None
def test_no_override_returns_none(self):
cfg = {"model": {"default": "my-llava"}}
assert _supports_vision_override(cfg, "custom", "my-llava") is None
def test_malformed_sections_are_ignored(self):
# User accidentally wrote a string where a section was expected —
# don't blow up, just fall through.
cfg = {"model": "some-string", "providers": ["not-a-dict"]}
assert _supports_vision_override(cfg, "custom", "my-llava") is None
# ─── _lookup_supports_vision (override-aware) ────────────────────────────────
class TestLookupSupportsVisionOverride:
def test_config_override_short_circuits_models_dev(self):
# Config says True, models.dev says None — config wins.
cfg = {"model": {"supports_vision": True}}
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert _lookup_supports_vision("custom", "my-llava", cfg) is True
def test_config_override_false_beats_vision_capable_models_dev(self):
# User explicitly disables vision on a models.dev-vision-capable model.
fake_caps = type("Caps", (), {"supports_vision": True})()
cfg = {"model": {"supports_vision": False}}
with patch("agent.models_dev.get_model_capabilities", return_value=fake_caps):
assert _lookup_supports_vision("anthropic", "claude-sonnet-4", cfg) is False
def test_no_override_falls_back_to_models_dev(self):
fake_caps = type("Caps", (), {"supports_vision": True})()
with patch("agent.models_dev.get_model_capabilities", return_value=fake_caps):
assert _lookup_supports_vision("anthropic", "claude-sonnet-4", {}) is True
def test_no_override_no_models_dev_entry_returns_none(self):
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert _lookup_supports_vision("custom", "my-llava", {}) is None
def test_cfg_none_falls_back_to_models_dev(self):
# Caller didn't pass cfg at all — old call sites must still work.
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert _lookup_supports_vision("openrouter", "x", None) is None
# ─── decide_image_input_mode with auto + override ────────────────────────────
class TestAutoModeRespectsOverride:
def test_auto_native_for_custom_with_supports_vision_true(self):
# The motivating bug: Qwen3.6 on local llama.cpp via provider=custom.
# Without the override, auto falls back to text. With it, auto picks
# native — no need to also set agent.image_input_mode: native.
cfg = {"model": {"supports_vision": True}}
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert decide_image_input_mode("custom", "qwen3.6-35b", cfg) == "native"
def test_auto_text_for_custom_with_supports_vision_false(self):
cfg = {"model": {"supports_vision": False}}
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert decide_image_input_mode("custom", "some-text-only", cfg) == "text"
def test_auto_text_for_custom_with_no_override(self):
# Unchanged baseline: unknown custom model → text.
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert decide_image_input_mode("custom", "unknown", {}) == "text"
def test_explicit_aux_vision_override_still_wins(self):
# If the user has configured a dedicated vision aux backend, respect
# it even when supports_vision: true is also set.
cfg = {
"model": {"supports_vision": True},
"auxiliary": {"vision": {"provider": "openrouter", "model": "gemini-2.5-pro"}},
}
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert decide_image_input_mode("custom", "qwen3.6-35b", cfg) == "text"
# ─── build_native_content_parts ──────────────────────────────────────────────
def _png_bytes() -> bytes:
"""Return a tiny valid 1x1 transparent PNG."""
return base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABpfZFQAAAAABJRU5ErkJggg=="
)
class TestBuildNativeContentParts:
def test_text_then_image(self, tmp_path: Path):
img = tmp_path / "cat.png"
img.write_bytes(_png_bytes())
parts, skipped = build_native_content_parts("hello", [str(img)])
assert skipped == []
assert len(parts) == 2
assert parts[0]["type"] == "text"
# User caption is preserved and a per-image path hint is appended so
# the model can use the local path as a string argument for tools
# that take ``image_url: str`` (issue #18960).
assert parts[0]["text"] == f"hello\n\n[Image attached at: {img}]"
assert parts[1]["type"] == "image_url"
assert parts[1]["image_url"]["url"].startswith("data:image/png;base64,")
def test_empty_text_inserts_default_prompt(self, tmp_path: Path):
img = tmp_path / "cat.jpg"
img.write_bytes(_png_bytes())
parts, skipped = build_native_content_parts("", [str(img)])
assert skipped == []
# Even with empty user text, we insert a neutral prompt so the turn
# isn't just pixels, and the path hint is appended after.
assert parts[0]["type"] == "text"
assert parts[0]["text"] == (
f"What do you see in this image?\n\n[Image attached at: {img}]"
)
assert parts[1]["type"] == "image_url"
def test_missing_file_is_skipped(self, tmp_path: Path):
parts, skipped = build_native_content_parts("hi", [str(tmp_path / "missing.png")])
assert skipped == [str(tmp_path / "missing.png")]
# Skipped paths are NOT advertised in the path hints — the model
# would otherwise be told a non-existent file is attached.
assert parts == [{"type": "text", "text": "hi"}]
def test_path_hint_appended(self, tmp_path: Path):
"""The local path of each attached image is appended to the user
text part so MCP/skill tools that take ``image_url: str`` can be
invoked on the same image (issue #18960). Mirrors text-mode
behaviour (`Runner._enrich_message_with_vision`).
"""
img = tmp_path / "scan.png"
img.write_bytes(_png_bytes())
parts, _ = build_native_content_parts("attach this", [str(img)])
text_part = next(p for p in parts if p.get("type") == "text")
assert "[Image attached at:" in text_part["text"]
assert str(img) in text_part["text"]
# User caption is preserved verbatim ahead of the hint.
assert text_part["text"].startswith("attach this")
def test_path_hint_one_per_attached_image(self, tmp_path: Path):
"""Each successfully attached image gets its own path hint line;
skipped images do NOT appear in the hints.
"""
good = tmp_path / "good.png"
good.write_bytes(_png_bytes())
missing = tmp_path / "missing.png" # never created
parts, skipped = build_native_content_parts(
"see attached", [str(good), str(missing)]
)
assert skipped == [str(missing)]
text_part = next(p for p in parts if p.get("type") == "text")
assert text_part["text"].count("[Image attached at:") == 1
assert str(good) in text_part["text"]
assert str(missing) not in text_part["text"]
def test_multiple_images(self, tmp_path: Path):
img1 = tmp_path / "a.png"
img2 = tmp_path / "b.png"
img1.write_bytes(_png_bytes())
img2.write_bytes(_png_bytes())
parts, skipped = build_native_content_parts("compare these", [str(img1), str(img2)])
assert skipped == []
image_parts = [p for p in parts if p.get("type") == "image_url"]
assert len(image_parts) == 2
# Both paths surface in the text part, one per line.
text_part = next(p for p in parts if p.get("type") == "text")
assert text_part["text"].count("[Image attached at:") == 2
assert str(img1) in text_part["text"]
assert str(img2) in text_part["text"]
def test_mime_inference_jpg(self, tmp_path: Path):
# Real JPEG bytes (SOI marker FF D8 FF): sniffing now wins over suffix.
img = tmp_path / "photo.jpg"
img.write_bytes(b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01" + b"\x00" * 32)
parts, _ = build_native_content_parts("x", [str(img)])
url = parts[1]["image_url"]["url"]
assert url.startswith("data:image/jpeg;base64,")
def test_mime_inference_webp(self, tmp_path: Path):
# Real WEBP bytes (RIFF....WEBP): sniffing now wins over suffix.
img = tmp_path / "pic.webp"
img.write_bytes(b"RIFF\x24\x00\x00\x00WEBPVP8 " + b"\x00" * 32)
parts, _ = build_native_content_parts("", [str(img)])
url = parts[1]["image_url"]["url"]
assert url.startswith("data:image/webp;base64,")
def test_mime_sniff_overrides_misleading_extension(self, tmp_path: Path):
"""Discord-style bug: file is named .webp but contains PNG bytes.
Anthropic rejects on MIME mismatch (HTTP 400) so we MUST sniff.
Regression guard for the user-reported Discord PNG-as-WEBP failure.
"""
img = tmp_path / "discord_cached.webp"
img.write_bytes(_png_bytes()) # bytes are PNG, suffix lies
parts, _ = build_native_content_parts("", [str(img)])
url = parts[1]["image_url"]["url"]
assert url.startswith("data:image/png;base64,"), (
f"Expected MIME sniffing to detect PNG bytes regardless of .webp suffix, got: {url[:60]}"
)
# ─── Oversize handling ───────────────────────────────────────────────────────
class TestLargeImageHandling:
"""Large images attach at native size; shrink is handled reactively at
retry time in ``run_agent._try_shrink_image_parts_in_messages`` rather
than proactively here.
"""
def test_large_image_passes_through_unchanged(self, tmp_path: Path):
"""A multi-MB image is attached as-is — no resize, no skip."""
from agent import image_routing as _ir
img = tmp_path / "medium.png"
# 200 KB of real bytes; not huge but enough to verify no size gate fires.
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"X" * 200_000)
url = _ir._file_to_data_url(img)
assert url is not None
assert url.startswith("data:image/png;base64,")
# Base64 expansion means output is ~4/3 of input, plus header.
assert len(url) > 200_000
def test_missing_file_returns_none(self, tmp_path: Path):
from agent import image_routing as _ir
missing = tmp_path / "does_not_exist.png"
assert _ir._file_to_data_url(missing) is None
def test_build_native_parts_no_provider_kwarg(self, tmp_path: Path):
"""build_native_content_parts takes text + paths, no provider kwarg."""
from agent import image_routing as _ir
img = tmp_path / "cat.png"
img.write_bytes(_png_bytes())
parts, skipped = _ir.build_native_content_parts("hi", [str(img)])
assert skipped == []
assert len(parts) == 2
assert parts[0]["type"] == "text"
assert parts[1]["type"] == "image_url"
# ─── extract_image_refs ──────────────────────────────────────────────────────
class TestExtractImageRefs:
"""Scan task body / inbound text for image paths and URLs (kanban worker
enrichment, issue raised May 2026)."""
def test_empty_or_none_returns_empty(self):
assert extract_image_refs("") == ([], [])
assert extract_image_refs(None) == ([], []) # type: ignore[arg-type]
def test_finds_absolute_path(self, tmp_path: Path):
img = tmp_path / "screenshot.png"
img.write_bytes(_png_bytes())
body = f"Look at {img} and tell me what's wrong."
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == []
def test_finds_home_relative_path(self, tmp_path: Path, monkeypatch):
# Simulate ~/foo.png by pointing HOME at tmp_path and creating the file
monkeypatch.setenv("HOME", str(tmp_path))
img = tmp_path / "foo.png"
img.write_bytes(_png_bytes())
paths, urls = extract_image_refs("see ~/foo.png please")
assert paths == [str(img)]
assert urls == []
def test_skips_nonexistent_paths(self, tmp_path: Path):
# Path-shaped but no file on disk → skipped.
body = f"What's at {tmp_path}/never_created.png ?"
paths, urls = extract_image_refs(body)
assert paths == []
assert urls == []
def test_finds_http_image_url(self):
body = "Check out https://example.com/photos/cat.png — cute right?"
paths, urls = extract_image_refs(body)
assert paths == []
assert urls == ["https://example.com/photos/cat.png"]
def test_finds_https_url_with_query_string(self):
body = "Diagram: https://cdn.example.com/img.jpeg?size=large&v=2 here"
paths, urls = extract_image_refs(body)
assert urls == ["https://cdn.example.com/img.jpeg?size=large&v=2"]
def test_url_trailing_punctuation_stripped(self):
# Prose punctuation right after the URL must not be part of the URL.
body = "See https://example.com/a.png."
paths, urls = extract_image_refs(body)
assert urls == ["https://example.com/a.png"]
def test_ignores_non_image_urls(self):
body = "See https://example.com/page.html and https://x.com/y.pdf"
paths, urls = extract_image_refs(body)
assert urls == []
def test_dedupes_paths_and_urls(self, tmp_path: Path):
img = tmp_path / "dup.png"
img.write_bytes(_png_bytes())
body = (
f"First {img} then again {img}. "
"Also https://example.com/x.png and https://example.com/x.png again."
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == ["https://example.com/x.png"]
def test_ignores_paths_in_fenced_code_block(self, tmp_path: Path):
img = tmp_path / "real.png"
img.write_bytes(_png_bytes())
body = (
"Outside the block, attach this:\n"
f"{img}\n"
"But not these examples:\n"
"```\n"
f"some_other_image: /tmp/example.png\n"
f"url: https://example.com/example.png\n"
"```\n"
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == []
def test_ignores_paths_in_inline_code(self, tmp_path: Path):
img = tmp_path / "real.jpg"
img.write_bytes(_png_bytes())
body = (
f"Attach {img}, but ignore the example "
"`https://example.com/skip.png` in backticks."
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == []
def test_does_not_match_paths_inside_urls(self, tmp_path: Path):
# The lookbehind in the regex prevents matching the path-portion of
# a URL as a local path. Only the URL should be detected.
body = "Just the URL: https://example.com/some/dir/image.png"
paths, urls = extract_image_refs(body)
assert paths == []
assert urls == ["https://example.com/some/dir/image.png"]
def test_mixed_paths_and_urls(self, tmp_path: Path):
img = tmp_path / "local.png"
img.write_bytes(_png_bytes())
body = (
f"Compare local {img} against the design at "
"https://example.com/design/v2.png — does it match?"
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == ["https://example.com/design/v2.png"]
def test_case_insensitive_extension(self, tmp_path: Path):
img = tmp_path / "shouty.PNG"
img.write_bytes(_png_bytes())
body = f"see {img}"
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
# ─── build_native_content_parts with URLs ────────────────────────────────────
class TestBuildNativeContentPartsURLs:
"""URL pass-through support added so kanban task bodies (and other
inbound surfaces) can route remote image URLs straight to the model."""
def test_url_only_no_local_paths(self):
parts, skipped = build_native_content_parts(
"what is this?",
[],
image_urls=["https://example.com/diagram.png"],
)
assert skipped == []
assert len(parts) == 2
assert parts[0]["type"] == "text"
assert "[Image attached: https://example.com/diagram.png]" in parts[0]["text"]
assert parts[0]["text"].startswith("what is this?")
assert parts[1] == {
"type": "image_url",
"image_url": {"url": "https://example.com/diagram.png"},
}
def test_mixed_path_and_url(self, tmp_path: Path):
img = tmp_path / "local.png"
img.write_bytes(_png_bytes())
parts, skipped = build_native_content_parts(
"compare these",
[str(img)],
image_urls=["https://example.com/remote.jpg"],
)
assert skipped == []
# 1 text + 2 image parts (local data URL first, then remote URL).
image_parts = [p for p in parts if p.get("type") == "image_url"]
assert len(image_parts) == 2
assert image_parts[0]["image_url"]["url"].startswith("data:image/png;base64,")
assert image_parts[1]["image_url"]["url"] == "https://example.com/remote.jpg"
text = parts[0]["text"]
assert "[Image attached at:" in text
assert "[Image attached: https://example.com/remote.jpg]" in text
def test_empty_url_list_is_no_op(self, tmp_path: Path):
img = tmp_path / "x.png"
img.write_bytes(_png_bytes())
# image_urls=[] should behave the same as not passing it at all.
parts_no_urls, _ = build_native_content_parts("hi", [str(img)])
parts_empty_urls, _ = build_native_content_parts("hi", [str(img)], image_urls=[])
assert parts_no_urls == parts_empty_urls
def test_blank_url_strings_are_dropped(self):
parts, _ = build_native_content_parts(
"x", [], image_urls=["", " ", "https://example.com/a.png"]
)
image_parts = [p for p in parts if p.get("type") == "image_url"]
assert len(image_parts) == 1
assert image_parts[0]["image_url"]["url"] == "https://example.com/a.png"
def test_url_only_inserts_default_prompt_when_text_empty(self):
parts, _ = build_native_content_parts(
"", [], image_urls=["https://example.com/a.png"]
)
assert parts[0]["type"] == "text"
assert parts[0]["text"].startswith("What do you see in this image?")
+730
View File
@@ -0,0 +1,730 @@
"""Tests for agent/insights.py — InsightsEngine analytics and reporting."""
import time
import pytest
from hermes_state import SessionDB
from agent.insights import (
InsightsEngine,
_estimate_cost,
_format_duration,
_bar_chart,
_has_known_pricing,
)
@pytest.fixture()
def db(tmp_path):
"""Create a SessionDB with a temp database file."""
db_path = tmp_path / "test_insights.db"
session_db = SessionDB(db_path=db_path)
yield session_db
session_db.close()
@pytest.fixture()
def populated_db(db):
"""Create a DB with realistic session data for insights testing."""
now = time.time()
day = 86400
# Session 1: CLI, claude-sonnet, ended, 2 days ago
db.create_session(
session_id="s1", source="cli",
model="anthropic/claude-sonnet-4-20250514", user_id="user1",
)
# Backdate the started_at
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's1'", (now - 2 * day,))
db.end_session("s1", end_reason="user_exit")
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's1'", (now - 2 * day + 3600,))
db.update_token_counts("s1", input_tokens=50000, output_tokens=15000)
db.append_message("s1", role="user", content="Hello, help me fix a bug")
db.append_message("s1", role="assistant", content="Sure, let me look into that.")
db.append_message("s1", role="assistant", content="Let me search the files.",
tool_calls=[{"function": {"name": "search_files"}}])
db.append_message("s1", role="tool", content="Found 3 matches", tool_name="search_files")
db.append_message("s1", role="assistant", content="Let me read the file.",
tool_calls=[{"function": {"name": "read_file"}}])
db.append_message("s1", role="tool", content="file contents...", tool_name="read_file")
db.append_message("s1", role="assistant", content="I found the bug. Let me fix it.",
tool_calls=[{"function": {"name": "patch"}}])
db.append_message("s1", role="tool", content="patched successfully", tool_name="patch")
db.append_message(
"s1",
role="assistant",
content="Let me load the PR workflow skill.",
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}],
)
db.append_message("s1", role="user", content="Thanks!")
db.append_message("s1", role="assistant", content="You're welcome!")
# Session 2: Telegram, gpt-4o, ended, 5 days ago
db.create_session(
session_id="s2", source="telegram",
model="gpt-4o", user_id="user1",
)
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's2'", (now - 5 * day,))
db.end_session("s2", end_reason="timeout")
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's2'", (now - 5 * day + 1800,))
db.update_token_counts("s2", input_tokens=20000, output_tokens=8000)
db.append_message("s2", role="user", content="Search the web for something")
db.append_message("s2", role="assistant", content="Searching...",
tool_calls=[{"function": {"name": "web_search"}}])
db.append_message("s2", role="tool", content="results...", tool_name="web_search")
db.append_message("s2", role="assistant", content="Here's what I found")
# Session 3: CLI, deepseek-chat, ended, 10 days ago
db.create_session(
session_id="s3", source="cli",
model="deepseek-chat", user_id="user1",
)
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's3'", (now - 10 * day,))
db.end_session("s3", end_reason="user_exit")
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's3'", (now - 10 * day + 7200,))
db.update_token_counts("s3", input_tokens=100000, output_tokens=40000)
db.append_message("s3", role="user", content="Run this terminal command")
db.append_message("s3", role="assistant", content="Running...",
tool_calls=[{"function": {"name": "terminal"}}])
db.append_message("s3", role="tool", content="output...", tool_name="terminal")
db.append_message("s3", role="assistant", content="Let me run another",
tool_calls=[{"function": {"name": "terminal"}}])
db.append_message("s3", role="tool", content="more output...", tool_name="terminal")
db.append_message("s3", role="assistant", content="And search files",
tool_calls=[{"function": {"name": "search_files"}}])
db.append_message("s3", role="tool", content="found stuff", tool_name="search_files")
db.append_message(
"s3",
role="assistant",
content="Load the debugging skill.",
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"systematic-debugging"}'}}],
)
# Session 4: Discord, same model as s1, ended, 1 day ago
db.create_session(
session_id="s4", source="discord",
model="anthropic/claude-sonnet-4-20250514", user_id="user2",
)
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's4'", (now - 1 * day,))
db.end_session("s4", end_reason="user_exit")
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's4'", (now - 1 * day + 900,))
db.update_token_counts("s4", input_tokens=10000, output_tokens=5000)
db.append_message("s4", role="user", content="Quick question")
db.append_message("s4", role="assistant", content="Sure, go ahead")
db.append_message(
"s4",
role="assistant",
content="Load and update GitHub skills.",
tool_calls=[
{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}},
{"function": {"name": "skill_manage", "arguments": '{"name":"github-code-review"}'}},
],
)
# Session 5: Old session, 45 days ago (should be excluded from 30-day window)
db.create_session(
session_id="s_old", source="cli",
model="gpt-4o-mini", user_id="user1",
)
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's_old'", (now - 45 * day,))
db.end_session("s_old", end_reason="user_exit")
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's_old'", (now - 45 * day + 600,))
db.update_token_counts("s_old", input_tokens=5000, output_tokens=2000)
db.append_message("s_old", role="user", content="old message")
db.append_message("s_old", role="assistant", content="old reply")
db._conn.commit()
return db
class TestHasKnownPricing:
def test_known_commercial_model(self):
assert _has_known_pricing("gpt-4o", provider="openai") is True
assert _has_known_pricing("anthropic/claude-sonnet-4-20250514") is True
assert _has_known_pricing("gpt-4.1", provider="openai") is True
def test_unknown_custom_model(self):
assert _has_known_pricing("FP16_Hermes_4.5") is False
assert _has_known_pricing("my-custom-model") is False
assert _has_known_pricing("glm-5") is False
assert _has_known_pricing("") is False
assert _has_known_pricing(None) is False
def test_heuristic_matched_models_are_not_considered_known(self):
assert _has_known_pricing("some-opus-model") is False
assert _has_known_pricing("future-sonnet-v2") is False
class TestEstimateCost:
def test_basic_cost(self):
cost, status = _estimate_cost(
"anthropic/claude-sonnet-4-20250514",
1_000_000,
1_000_000,
provider="anthropic",
)
assert status == "estimated"
assert cost == pytest.approx(18.0, abs=0.01)
def test_zero_tokens(self):
cost, status = _estimate_cost("gpt-4o", 0, 0, provider="openai")
assert status == "estimated"
assert cost == 0.0
def test_cache_aware_usage(self):
cost, status = _estimate_cost(
"anthropic/claude-sonnet-4-20250514",
1000,
500,
cache_read_tokens=2000,
cache_write_tokens=400,
provider="anthropic",
)
assert status == "estimated"
expected = (1000 * 3.0 + 500 * 15.0 + 2000 * 0.30 + 400 * 3.75) / 1_000_000
assert cost == pytest.approx(expected, abs=0.0001)
# =========================================================================
# Format helpers
# =========================================================================
class TestFormatDuration:
def test_seconds(self):
assert _format_duration(45) == "45s"
def test_minutes(self):
assert _format_duration(300) == "5m"
def test_hours_with_minutes(self):
result = _format_duration(5400) # 1.5 hours
assert result == "1h 30m"
def test_exact_hours(self):
assert _format_duration(7200) == "2h"
def test_days(self):
result = _format_duration(172800) # 2 days
assert result == "2.0d"
class TestBarChart:
def test_basic_bars(self):
bars = _bar_chart([10, 5, 0, 20], max_width=10)
assert len(bars) == 4
assert len(bars[3]) == 10 # max value gets full width
assert len(bars[0]) == 5 # half of max
assert bars[2] == "" # zero gets empty
def test_empty_values(self):
bars = _bar_chart([], max_width=10)
assert bars == []
def test_all_zeros(self):
bars = _bar_chart([0, 0, 0], max_width=10)
assert all(b == "" for b in bars)
def test_single_value(self):
bars = _bar_chart([5], max_width=10)
assert len(bars) == 1
assert len(bars[0]) == 10
# =========================================================================
# InsightsEngine — empty DB
# =========================================================================
class TestInsightsEmpty:
def test_empty_db_returns_empty_report(self, db):
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report["empty"] is True
assert report["overview"] == {}
def test_empty_db_terminal_format(self, db):
engine = InsightsEngine(db)
report = engine.generate(days=30)
text = engine.format_terminal(report)
assert "No sessions found" in text
def test_empty_db_gateway_format(self, db):
engine = InsightsEngine(db)
report = engine.generate(days=30)
text = engine.format_gateway(report)
assert "No sessions found" in text
# =========================================================================
# InsightsEngine — populated DB
# =========================================================================
class TestInsightsPopulated:
def test_generate_returns_all_sections(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
assert report["empty"] is False
assert "overview" in report
assert "models" in report
assert "platforms" in report
assert "tools" in report
assert "activity" in report
assert "top_sessions" in report
def test_overview_session_count(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
overview = report["overview"]
# s1, s2, s3, s4 are within 30 days; s_old is 45 days ago
assert overview["total_sessions"] == 4
def test_overview_token_totals(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
overview = report["overview"]
expected_input = 50000 + 20000 + 100000 + 10000
expected_output = 15000 + 8000 + 40000 + 5000
assert overview["total_input_tokens"] == expected_input
assert overview["total_output_tokens"] == expected_output
assert overview["total_tokens"] == expected_input + expected_output
def test_overview_cost_positive(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
assert report["overview"]["estimated_cost"] > 0
def test_overview_duration_stats(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
overview = report["overview"]
# All 4 sessions have durations
assert overview["total_hours"] > 0
assert overview["avg_session_duration"] > 0
def test_model_breakdown(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
models = report["models"]
# Should have 3 distinct models (claude-sonnet x2, gpt-4o, deepseek-chat)
model_names = [m["model"] for m in models]
assert "claude-sonnet-4-20250514" in model_names
assert "gpt-4o" in model_names
assert "deepseek-chat" in model_names
# Claude-sonnet has 2 sessions (s1 + s4)
claude = next(m for m in models if "claude-sonnet" in m["model"])
assert claude["sessions"] == 2
def test_platform_breakdown(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
platforms = report["platforms"]
platform_names = [p["platform"] for p in platforms]
assert "cli" in platform_names
assert "telegram" in platform_names
assert "discord" in platform_names
cli = next(p for p in platforms if p["platform"] == "cli")
assert cli["sessions"] == 2 # s1 + s3
def test_tool_breakdown(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
tools = report["tools"]
tool_names = [t["tool"] for t in tools]
assert "terminal" in tool_names
assert "search_files" in tool_names
assert "read_file" in tool_names
assert "patch" in tool_names
assert "web_search" in tool_names
# terminal was used 2x in s3
terminal = next(t for t in tools if t["tool"] == "terminal")
assert terminal["count"] == 2
# Percentages should sum to ~100%
total_pct = sum(t["percentage"] for t in tools)
assert total_pct == pytest.approx(100.0, abs=0.1)
def test_skill_breakdown(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
skills = report["skills"]
assert skills["summary"]["distinct_skills_used"] == 3
assert skills["summary"]["total_skill_loads"] == 3
assert skills["summary"]["total_skill_edits"] == 1
assert skills["summary"]["total_skill_actions"] == 4
top_skill = skills["top_skills"][0]
assert top_skill["skill"] == "github-pr-workflow"
assert top_skill["view_count"] == 2
assert top_skill["manage_count"] == 0
assert top_skill["total_count"] == 2
assert top_skill["last_used_at"] is not None
def test_skill_breakdown_respects_days_filter(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=3)
skills = report["skills"]
assert skills["summary"]["distinct_skills_used"] == 2
assert skills["summary"]["total_skill_loads"] == 2
assert skills["summary"]["total_skill_edits"] == 1
skill_names = [s["skill"] for s in skills["top_skills"]]
assert "systematic-debugging" not in skill_names
def test_activity_patterns(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
activity = report["activity"]
assert len(activity["by_day"]) == 7
assert len(activity["by_hour"]) == 24
assert activity["active_days"] >= 1
assert activity["busiest_day"] is not None
assert activity["busiest_hour"] is not None
def test_top_sessions(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
top = report["top_sessions"]
labels = [t["label"] for t in top]
assert "Longest session" in labels
assert "Most messages" in labels
assert "Most tokens" in labels
assert "Most tool calls" in labels
def test_source_filter_cli(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30, source="cli")
assert report["overview"]["total_sessions"] == 2 # s1, s3
def test_source_filter_telegram(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30, source="telegram")
assert report["overview"]["total_sessions"] == 1 # s2
def test_source_filter_nonexistent(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30, source="slack")
assert report["empty"] is True
def test_days_filter_short(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=3)
# Only s1 (2 days ago) and s4 (1 day ago) should be included
assert report["overview"]["total_sessions"] == 2
def test_days_filter_long(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=60)
# All 5 sessions should be included
assert report["overview"]["total_sessions"] == 5
# =========================================================================
# Formatting
# =========================================================================
class TestTerminalFormatting:
def test_terminal_format_has_sections(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_terminal(report)
assert "Hermes Insights" in text
assert "Overview" in text
assert "Models Used" in text
assert "Top Tools" in text
assert "Top Skills" in text
assert "Activity Patterns" in text
assert "Notable Sessions" in text
def test_terminal_format_shows_tokens(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_terminal(report)
assert "Input tokens" in text
assert "Output tokens" in text
# Cost and cache metrics are intentionally hidden (pricing was unreliable).
assert "Est. cost" not in text
assert "Cache read" not in text
assert "Cache write" not in text
def test_terminal_format_shows_platforms(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_terminal(report)
# Multi-platform, so Platforms section should show
assert "Platforms" in text
assert "cli" in text
assert "telegram" in text
def test_terminal_format_shows_bar_chart(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_terminal(report)
assert "█" in text # Bar chart characters
def test_terminal_format_hides_cost_for_custom_models(self, db):
"""Cost display is hidden entirely — custom models no longer show 'N/A' either."""
db.create_session(session_id="s1", source="cli", model="my-custom-model")
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
text = engine.format_terminal(report)
assert "N/A" not in text
assert "custom/self-hosted" not in text
assert "Cost" not in text
class TestGatewayFormatting:
def test_gateway_format_is_shorter(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
terminal_text = engine.format_terminal(report)
gateway_text = engine.format_gateway(report)
assert len(gateway_text) < len(terminal_text)
def test_gateway_format_has_bold(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_gateway(report)
assert "**" in text # Markdown bold
def test_gateway_format_hides_cost(self, populated_db):
"""Gateway format omits dollar figures and internal cache details."""
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_gateway(report)
assert "$" not in text
assert "cache" not in text.lower()
def test_gateway_format_shows_models(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
text = engine.format_gateway(report)
assert "Models" in text
assert "sessions" in text
# =========================================================================
# Edge cases
# =========================================================================
class TestEdgeCases:
def test_session_with_no_tokens(self, db):
"""Sessions with zero tokens should not crash."""
db.create_session(session_id="s1", source="cli", model="test-model")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report["empty"] is False
assert report["overview"]["total_tokens"] == 0
assert report["overview"]["estimated_cost"] == 0.0
def test_session_with_no_end_time(self, db):
"""Active (non-ended) sessions should be included but duration = 0."""
db.create_session(session_id="s1", source="cli", model="test-model")
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
# Session included
assert report["overview"]["total_sessions"] == 1
assert report["overview"]["total_tokens"] == 1500
# But no duration stats (session not ended)
assert report["overview"]["total_hours"] == 0
def test_session_with_no_model(self, db):
"""Sessions with NULL model should not crash."""
db.create_session(session_id="s1", source="cli")
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report["empty"] is False
models = report["models"]
assert len(models) == 1
assert models[0]["model"] == "unknown"
assert models[0]["has_pricing"] is False
def test_custom_model_shows_zero_cost(self, db):
"""Custom/self-hosted models should show $0 cost, not fake estimates."""
db.create_session(session_id="s1", source="cli", model="FP16_Hermes_4.5")
db.update_token_counts("s1", input_tokens=100000, output_tokens=50000)
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report["overview"]["estimated_cost"] == 0.0
assert "FP16_Hermes_4.5" in report["overview"]["models_without_pricing"]
models = report["models"]
custom = next(m for m in models if m["model"] == "FP16_Hermes_4.5")
assert custom["cost"] == 0.0
assert custom["has_pricing"] is False
def test_tool_usage_from_tool_calls_json(self, db):
"""Tool usage should be extracted from tool_calls JSON when tool_name is NULL."""
db.create_session(session_id="s1", source="cli", model="test")
# Assistant message with tool_calls (this is what CLI produces)
db.append_message("s1", role="assistant", content="Let me search",
tool_calls=[{"id": "call_1", "type": "function",
"function": {"name": "search_files", "arguments": "{}"}}])
# Tool response WITHOUT tool_name (this is the CLI bug)
db.append_message("s1", role="tool", content="found results",
tool_call_id="call_1")
db.append_message("s1", role="assistant", content="Now reading",
tool_calls=[{"id": "call_2", "type": "function",
"function": {"name": "read_file", "arguments": "{}"}}])
db.append_message("s1", role="tool", content="file content",
tool_call_id="call_2")
db.append_message("s1", role="assistant", content="And searching again",
tool_calls=[{"id": "call_3", "type": "function",
"function": {"name": "search_files", "arguments": "{}"}}])
db.append_message("s1", role="tool", content="more results",
tool_call_id="call_3")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
tools = report["tools"]
# Should find tools from tool_calls JSON even though tool_name is NULL
tool_names = [t["tool"] for t in tools]
assert "search_files" in tool_names
assert "read_file" in tool_names
# search_files was called twice
sf = next(t for t in tools if t["tool"] == "search_files")
assert sf["count"] == 2
def test_overview_pricing_sets_are_lists(self, db):
"""models_with/without_pricing should be JSON-serializable lists."""
import json as _json
db.create_session(session_id="s1", source="cli", model="gpt-4o")
db.create_session(session_id="s2", source="cli", model="my-custom")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
overview = report["overview"]
assert isinstance(overview["models_with_pricing"], list)
assert isinstance(overview["models_without_pricing"], list)
# Should be JSON-serializable
_json.dumps(report["overview"]) # would raise if sets present
def test_mixed_commercial_and_custom_models(self, db):
"""Mix of commercial and custom models: only commercial ones get costs."""
db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4-20250514")
db.update_token_counts(
"s1",
input_tokens=10000,
output_tokens=5000,
billing_provider="anthropic",
)
db.create_session(session_id="s2", source="cli", model="my-local-llama")
db.update_token_counts("s2", input_tokens=10000, output_tokens=5000)
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
# Cost should only come from gpt-4o, not from the custom model
overview = report["overview"]
assert overview["estimated_cost"] > 0
assert "claude-sonnet-4-20250514" in overview["models_with_pricing"] # list now, not set
assert "my-local-llama" in overview["models_without_pricing"]
# Verify individual model entries
claude = next(m for m in report["models"] if m["model"] == "claude-sonnet-4-20250514")
assert claude["has_pricing"] is True
assert claude["cost"] > 0
llama = next(m for m in report["models"] if m["model"] == "my-local-llama")
assert llama["has_pricing"] is False
assert llama["cost"] == 0.0
def test_single_session_streak(self, db):
"""Single session should have streak of 0 or 1."""
db.create_session(session_id="s1", source="cli", model="test")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report["activity"]["max_streak"] <= 1
def test_no_tool_calls(self, db):
"""Sessions with no tool calls should produce empty tools list."""
db.create_session(session_id="s1", source="cli", model="test")
db.append_message("s1", role="user", content="hello")
db.append_message("s1", role="assistant", content="hi there")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report["tools"] == []
def test_only_one_platform(self, db):
"""Single-platform usage should still work."""
db.create_session(session_id="s1", source="cli", model="test")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert len(report["platforms"]) == 1
assert report["platforms"][0]["platform"] == "cli"
# Terminal format should NOT show platform section for single platform
text = engine.format_terminal(report)
# (it still shows platforms section if there's only cli and nothing else)
# Actually the condition is > 1 platforms OR non-cli, so single cli won't show
def test_large_days_value(self, db):
"""Very large days value should not crash."""
db.create_session(session_id="s1", source="cli", model="test")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=365)
assert report["empty"] is False
def test_zero_days(self, db):
"""Zero days should return empty (nothing is in the future)."""
db.create_session(session_id="s1", source="cli", model="test")
db._conn.commit()
engine = InsightsEngine(db)
report = engine.generate(days=0)
# Depending on timing, might catch the session if created <1s ago
# Just verify it doesn't crash
assert "empty" in report
+25
View File
@@ -0,0 +1,25 @@
from __future__ import annotations
import importlib
import sys
from agent import jiter_preload
def test_preload_jiter_native_extension_loads_sdk_parser_dependency():
assert jiter_preload.preload_jiter_native_extension() is True
assert "jiter.jiter" in sys.modules
def test_preload_jiter_native_extension_is_best_effort(monkeypatch):
monkeypatch.setattr(jiter_preload, "_JITER_PRELOADED", False)
def _raise_missing(name: str):
assert name == "jiter.jiter"
raise ModuleNotFoundError(name)
monkeypatch.setattr(importlib, "import_module", _raise_missing)
assert jiter_preload.preload_jiter_native_extension() is False
assert jiter_preload._JITER_PRELOADED is False
assert isinstance(jiter_preload._JITER_PRELOAD_ERROR, ModuleNotFoundError)
@@ -0,0 +1,212 @@
"""Regression guard: don't send Anthropic ``thinking`` to Kimi's /coding endpoint.
Kimi's ``api.kimi.com/coding`` endpoint speaks the Anthropic Messages protocol
but has its own thinking semantics. When ``thinking.enabled`` is present in
the request, Kimi validates the message history and requires every prior
assistant tool-call message to carry OpenAI-style ``reasoning_content``.
The Anthropic path never populates that field, and
``convert_messages_to_anthropic`` strips Anthropic thinking blocks on
third-party endpoints — so after one turn with tool calls the next request
fails with HTTP 400::
thinking is enabled but reasoning_content is missing in assistant
tool call message at index N
Kimi on the chat_completions route handles ``thinking`` via ``extra_body`` in
``ChatCompletionsTransport`` (#13503). On the Anthropic route the right
thing to do is drop the parameter entirely and let Kimi drive reasoning
server-side.
"""
from __future__ import annotations
import pytest
class TestKimiCodingSkipsAnthropicThinking:
"""build_anthropic_kwargs must not inject ``thinking`` for Kimi /coding."""
@pytest.mark.parametrize(
"base_url",
[
"https://api.kimi.com/coding",
"https://api.kimi.com/coding/v1",
"https://api.kimi.com/coding/anthropic",
"https://api.kimi.com/coding/",
],
)
def test_kimi_coding_endpoint_omits_thinking(self, base_url: str) -> None:
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="kimi-k2.5",
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": True, "effort": "medium"},
base_url=base_url,
)
assert "thinking" not in kwargs, (
"Anthropic thinking must not be sent to Kimi /coding — "
"endpoint requires reasoning_content on history we don't preserve."
)
assert "output_config" not in kwargs
def test_kimi_coding_with_explicit_disabled_also_omits(self) -> None:
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="kimi-k2.5",
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": False},
base_url="https://api.kimi.com/coding",
)
assert "thinking" not in kwargs
def test_non_kimi_third_party_still_gets_thinking(self) -> None:
"""MiniMax and other third-party Anthropic endpoints must retain thinking."""
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="MiniMax-M2.7",
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": True, "effort": "medium"},
base_url="https://api.minimax.io/anthropic",
)
assert "thinking" in kwargs
assert kwargs["thinking"]["type"] == "enabled"
def test_native_anthropic_still_gets_thinking(self) -> None:
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": True, "effort": "medium"},
base_url=None,
)
assert "thinking" in kwargs
def test_kimi_root_endpoint_via_anthropic_transport_omits_thinking(self) -> None:
"""Plain ``api.kimi.com`` hit via the Anthropic transport also omits thinking.
Auto-detection routes ``api.kimi.com/v1`` to ``chat_completions`` by
default, but users can explicitly configure
``api_mode: anthropic_messages`` against any Kimi host. The upstream
validation (reasoning_content required on replayed tool-call
messages) is the same regardless of URL path, so the thinking
suppression must apply to every Kimi host, not just ``/coding``.
See #17057.
"""
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="kimi-k2.5",
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": True, "effort": "medium"},
base_url="https://api.kimi.com/v1",
)
assert "thinking" not in kwargs
# ── #17057: custom / proxied Kimi-compatible endpoints ──────────
@pytest.mark.parametrize(
"base_url,model",
[
# Custom host with Kimi-family model — the reporter's case
("http://my-kimi-proxy.internal", "kimi-2.6"),
("https://llm.example.com/anthropic", "kimi-k2.5"),
("https://llm.example.com/anthropic", "moonshot-v1-8k"),
("https://llm.example.com/anthropic", "kimi_thinking"),
("https://llm.example.com/anthropic", "moonshotai/kimi-k2.5"),
# Official Moonshot host (previously uncovered)
("https://api.moonshot.ai/anthropic", "moonshot-v1-32k"),
("https://api.moonshot.cn/anthropic", "moonshot-v1-32k"),
],
)
def test_kimi_family_custom_endpoint_omits_thinking(
self, base_url: str, model: str
) -> None:
"""Custom / proxied Kimi endpoints must also strip Anthropic thinking."""
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model=model,
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": True, "effort": "medium"},
base_url=base_url,
)
assert "thinking" not in kwargs, (
f"Kimi-family endpoint ({base_url}, {model}) must not receive "
f"Anthropic thinking — upstream validates reasoning_content on "
f"replayed tool-call history we don't preserve."
)
assert "output_config" not in kwargs
def test_custom_endpoint_non_kimi_model_keeps_thinking(self) -> None:
"""Custom endpoint with a non-Kimi model must keep thinking intact.
Guards against over-broad model-family matching — only model names
starting with a Kimi/Moonshot prefix should trigger suppression.
"""
from agent.anthropic_adapter import build_anthropic_kwargs
kwargs = build_anthropic_kwargs(
model="MiniMax-M2.7",
messages=[{"role": "user", "content": "hello"}],
tools=None,
max_tokens=4096,
reasoning_config={"enabled": True, "effort": "medium"},
base_url="https://my-llm-proxy.example.com/anthropic",
)
assert "thinking" in kwargs
assert kwargs["thinking"]["type"] == "enabled"
def test_kimi_family_replay_preserves_unsigned_thinking(self) -> None:
"""On a custom Kimi endpoint, unsigned reasoning_content thinking
blocks must survive the third-party signature-stripping pass so
the upstream's message-history validation passes.
"""
from agent.anthropic_adapter import convert_messages_to_anthropic
messages = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"reasoning_content": "planning the tool call",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "skill_view", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
]
_, converted = convert_messages_to_anthropic(
messages,
base_url="http://my-kimi-proxy.internal",
model="kimi-2.6",
)
# The assistant message still carries the unsigned thinking block
# synthesised from reasoning_content (required by Kimi's history
# validation). A plain third-party endpoint would have stripped it.
assistant_msg = next(m for m in converted if m["role"] == "assistant")
assistant_blocks = assistant_msg["content"]
thinking_blocks = [
b for b in assistant_blocks
if isinstance(b, dict) and b.get("type") == "thinking"
]
assert len(thinking_blocks) == 1
assert thinking_blocks[0]["thinking"] == "planning the tool call"
+22
View File
@@ -0,0 +1,22 @@
"""Test that last_total_tokens is correctly set by ContextCompressor."""
from agent.context_compressor import ContextCompressor
def test_update_from_response_sets_total_tokens():
"""ABC contract: last_total_tokens must be set from API response."""
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30, "total_tokens": 130})
assert c.last_total_tokens == 130
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30})
assert c.last_total_tokens == 130
def test_session_reset_clears_total_tokens():
"""on_session_reset must zero total_tokens."""
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30, "total_tokens": 130})
c.on_session_reset()
assert c.last_total_tokens == 0
+130
View File
@@ -0,0 +1,130 @@
"""Tests for local provider stream read timeout auto-detection.
When a local LLM provider is detected (Ollama, llama.cpp, vLLM, etc.),
the httpx stream read timeout should be automatically increased from the
default 60s to HERMES_API_TIMEOUT (1800s) to avoid premature connection
kills during long prefill phases.
"""
import os
import pytest
from unittest.mock import patch
from agent.model_metadata import is_local_endpoint
class TestLocalStreamReadTimeout:
"""Verify stream read timeout auto-detection logic."""
@pytest.mark.parametrize("base_url", [
"http://localhost:11434",
"http://127.0.0.1:8080",
"http://0.0.0.0:5000",
"http://192.168.1.100:8000",
"http://10.0.0.5:1234",
"http://host.docker.internal:11434",
"http://host.containers.internal:11434",
"http://host.lima.internal:11434",
])
def test_local_endpoint_bumps_read_timeout(self, base_url):
"""Local endpoint + default timeout -> bumps to base_timeout."""
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("HERMES_STREAM_READ_TIMEOUT", None)
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
_stream_read_timeout = _base_timeout
assert _stream_read_timeout == 1800.0
def test_user_override_respected_for_local(self):
"""User sets HERMES_STREAM_READ_TIMEOUT -> keep their value even for local."""
with patch.dict(os.environ, {"HERMES_STREAM_READ_TIMEOUT": "300"}, clear=False):
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
base_url = "http://localhost:11434"
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
_stream_read_timeout = _base_timeout
assert _stream_read_timeout == 300.0
@pytest.mark.parametrize("base_url", [
"https://api.openai.com",
"https://openrouter.ai/api",
"https://api.anthropic.com",
])
def test_remote_endpoint_keeps_default(self, base_url):
"""Remote endpoint -> keep 120s default."""
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("HERMES_STREAM_READ_TIMEOUT", None)
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
_stream_read_timeout = _base_timeout
assert _stream_read_timeout == 120.0
def test_empty_base_url_keeps_default(self):
"""No base_url set -> keep 120s default."""
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("HERMES_STREAM_READ_TIMEOUT", None)
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
base_url = ""
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
_stream_read_timeout = _base_timeout
assert _stream_read_timeout == 120.0
class TestIsLocalEndpoint:
"""Direct unit tests for is_local_endpoint."""
@pytest.mark.parametrize("url", [
"http://localhost:11434",
"http://127.0.0.1:8080",
"http://0.0.0.0:5000",
"http://[::1]:11434",
"http://192.168.1.100:8000",
"http://10.0.0.5:1234",
"http://172.17.0.1:11434",
])
def test_classic_local_addresses(self, url):
assert is_local_endpoint(url) is True
@pytest.mark.parametrize("url", [
"http://host.docker.internal:11434",
"http://host.docker.internal:8080/v1",
"http://gateway.docker.internal:11434",
"http://host.containers.internal:11434",
"http://host.lima.internal:11434",
])
def test_container_dns_names(self, url):
assert is_local_endpoint(url) is True
@pytest.mark.parametrize("url", [
"https://api.openai.com",
"https://openrouter.ai/api",
"https://api.anthropic.com",
"https://evil.docker.internal.example.com",
])
def test_remote_endpoints(self, url):
assert is_local_endpoint(url) is False
@pytest.mark.parametrize("url", [
"http://100.64.0.0:11434", # lower bound of CGNAT block
"http://100.64.0.1:11434/v1", # lower bound +1
"http://100.77.243.5:11434", # representative Tailscale host
"https://100.100.100.100:443", # Tailscale MagicDNS anchor
"https://100.127.255.254:443", # upper bound -1
"http://100.127.255.255:11434", # upper bound of CGNAT block
])
def test_tailscale_cgnat_is_local(self, url):
"""Tailscale 100.64.0.0/10 should be treated as local for timeout bumps."""
assert is_local_endpoint(url) is True
@pytest.mark.parametrize("url", [
"http://100.63.255.255:11434", # just below CGNAT block
"http://100.128.0.1:11434", # just above CGNAT block
"http://100.200.0.1:11434", # well outside CGNAT
"http://99.64.0.1:11434", # first octet wrong
])
def test_near_but_not_cgnat_is_remote(self, url):
"""Hosts adjacent to but outside 100.64.0.0/10 must not match."""
assert is_local_endpoint(url) is False
+312
View File
@@ -0,0 +1,312 @@
"""Tests for `agent.markdown_tables.realign_markdown_tables`.
These cover the alignment guarantee on CJK / wide-character tables and
the conservative no-op behaviour on non-table input.
"""
from __future__ import annotations
from textwrap import dedent
from wcwidth import wcswidth
from agent.markdown_tables import (
is_table_divider,
looks_like_table_row,
realign_markdown_tables,
split_table_row,
)
def _column_offsets(line: str) -> list[int]:
"""Return the display-cell index of every ``|`` in ``line``."""
cells: list[int] = []
width = 0
for ch in line:
if ch == "|":
cells.append(width)
# wcswidth on a single char; clamp negatives.
w = wcswidth(ch)
width += w if w > 0 else 1
return cells
# ---------------------------------------------------------------------------
# split_table_row / is_table_divider / looks_like_table_row
# ---------------------------------------------------------------------------
def test_split_strips_outer_pipes_and_trims():
assert split_table_row("| a | b | c |") == ["a", "b", "c"]
assert split_table_row("|配置|状态|") == ["配置", "状态"]
assert split_table_row("a | b | c") == ["a", "b", "c"]
def test_is_table_divider_handles_alignment_colons():
assert is_table_divider("|---|---|")
assert is_table_divider("| :--- | ---: | :---: |")
assert not is_table_divider("| - | - |") # 1 dash is not a divider
assert not is_table_divider("| a | b |")
assert not is_table_divider("---") # single column, no pipes
def test_looks_like_table_row():
assert looks_like_table_row("| a | b |")
assert looks_like_table_row("a | b | c") # no leading pipe, ≥2 pipes
assert not looks_like_table_row("not a table")
assert not looks_like_table_row("a | b") # one pipe, no leading pipe
assert not looks_like_table_row("")
# ---------------------------------------------------------------------------
# realign_markdown_tables
# ---------------------------------------------------------------------------
def test_no_op_on_text_without_tables():
text = "Hello world\nThis has no | pipes table.\n"
assert realign_markdown_tables(text) == text
def test_no_op_when_pipes_but_no_divider():
text = "echo a | grep b\necho c | wc -l\n"
assert realign_markdown_tables(text) == text
def test_cjk_table_pipes_align_across_rows():
# Model-emitted (under-padded for CJK) input.
src = dedent(
"""\
| 配置 | Config | 论文 (%) | 复现 (%) | 差值 | 状态 |
|------|--------|---------|---------|------|------|
| Vicuna (report) | dense | 79.30 | 未完成 | - | × |
| ChatGLM | chat | 37.60 | 37.82 | +0.22 | ✓ |
| 通义千问 | qwen | (无) | 报错 | - | × |
"""
)
out = realign_markdown_tables(src).rstrip("\n").split("\n")
# All rows in the rebuilt block must have pipes at identical display
# columns — that's the alignment guarantee.
offsets = [_column_offsets(row) for row in out]
assert all(o == offsets[0] for o in offsets), (
"rebuilt table rows do not share pipe column offsets:\n"
+ "\n".join(out)
)
# And we expect 7 pipes per row (6 columns + outer borders).
assert len(offsets[0]) == 7
def test_emoji_with_cjk_table_aligns():
src = dedent(
"""\
| 模型 | 状态 | 备注 |
|------|------|------|
| 千问 | ✅ | 通过 |
| Claude | ✅ | 推理强 |
| 文心一言 | ❌ | 报错 |
"""
)
out = realign_markdown_tables(src).rstrip("\n").split("\n")
offsets = [_column_offsets(row) for row in out]
# The emoji-with-variation-selector case (⚠️) intentionally tolerates
# 1-cell drift; bare emoji like ✅ / ❌ have stable wcwidth and must
# align. Use bare emoji here so the assertion is hard.
assert all(o == offsets[0] for o in offsets), (
"emoji+CJK rows do not share pipe column offsets:\n" + "\n".join(out)
)
def test_already_aligned_ascii_table_remains_aligned():
src = dedent(
"""\
| a | b |
|-----|-----|
| 1 | 2 |
| foo | bar |
"""
)
out = realign_markdown_tables(src).rstrip("\n").split("\n")
offsets = [_column_offsets(row) for row in out]
assert all(o == offsets[0] for o in offsets)
def test_passes_non_table_lines_through_around_a_table():
src = dedent(
"""\
Here is a comparison:
| 模型 | 状态 |
|------|------|
| 千问 | 通过 |
And some prose after.
"""
)
out = realign_markdown_tables(src)
assert out.startswith("Here is a comparison:\n")
assert out.endswith("And some prose after.\n")
# And the table lines are aligned.
block = [ln for ln in out.split("\n") if "|" in ln]
offsets = [_column_offsets(row) for row in block]
assert all(o == offsets[0] for o in offsets)
# ---------------------------------------------------------------------------
# Vertical fallback for tables wider than the terminal
# ---------------------------------------------------------------------------
def test_overflow_falls_back_to_vertical_when_table_wider_than_terminal():
"""A horizontal table that would exceed the available width must
drop to vertical key-value rendering so the terminal does not
soft-wrap mid-cell (which destroys column alignment visually)."""
src = dedent(
"""\
| Item | Description | Notes |
|------|-------------|-------|
| a | short | ok |
| b | this is a much longer description that stretches the column wider than the others by a lot | fine |
| c | tiny | - |
"""
)
out = realign_markdown_tables(src, available_width=100)
# No horizontal pipe-bordered rows: vertical mode emits "Header: value"
# lines and a ─ separator instead.
assert "|" not in out
assert "Item: a" in out
assert "Description: short" in out
assert "Notes: ok" in out
# Body rows separated by ─ rule
assert "──" in out
# Every emitted line fits the available width.
for line in out.split("\n"):
assert wcswidth(line) <= 100, f"line wider than budget: {line!r}"
def test_horizontal_kept_when_table_fits():
"""A table that fits the terminal must keep the horizontal
pipe-bordered rendering — vertical fallback only kicks in when
soft-wrap is unavoidable."""
src = dedent(
"""\
| Name | Age |
|------|-----|
| Alice | 30 |
| Bob | 25 |
"""
)
out = realign_markdown_tables(src, available_width=100)
# Pipe-bordered rendering survives.
body_rows = [ln for ln in out.split("\n") if ln.strip().startswith("|")]
assert len(body_rows) == 4
offsets = [_column_offsets(r) for r in body_rows]
assert all(o == offsets[0] for o in offsets)
def test_vertical_fallback_wraps_long_cell_text_with_indent():
src = dedent(
"""\
| Key | Value |
|-----|-------|
| x | this value is long enough that wrapping the value to fit a narrow terminal width is required even in vertical mode |
"""
)
out = realign_markdown_tables(src, available_width=60)
lines = out.split("\n")
assert lines[0].startswith("Key: x")
# First "Value:" line + at least one continuation indented by 2 spaces.
value_idx = next(i for i, l in enumerate(lines) if l.startswith("Value:"))
assert lines[value_idx + 1].startswith(" ")
# Every line still fits the budget.
for line in lines:
assert wcswidth(line) <= 60
def test_overflow_falls_back_to_vertical_for_cjk_too():
"""CJK content can also push a table over the terminal budget;
the vertical fallback should kick in regardless of script."""
src = dedent(
"""\
| 模型 | 描述 | 备注 |
|------|------|------|
| 千问 | 一个相当长的描述用于把列宽撑得超过可用终端宽度从而触发竖排回退 | 通过 |
| 文心 | 短 | × |
"""
)
out = realign_markdown_tables(src, available_width=50)
assert "|" not in out
assert "模型: 千问" in out
assert "模型: 文心" in out
for line in out.split("\n"):
assert wcswidth(line) <= 50, f"line wider than budget: {line!r}"
def test_handles_ragged_rows_by_padding_short_rows():
src = dedent(
"""\
| a | b | c |
|---|---|---|
| 1 | 2 |
| x | y | z |
"""
)
out = realign_markdown_tables(src).rstrip("\n").split("\n")
offsets = [_column_offsets(row) for row in out]
# Short rows must be padded out so they have the same pipe count
# and column positions as the header.
assert all(len(o) == len(offsets[0]) for o in offsets)
assert all(o == offsets[0] for o in offsets)
def test_multiple_tables_in_one_text():
src = dedent(
"""\
First:
| 配置 | 值 |
|------|----|
| 通义 | 1 |
Second:
| model | n |
|-------|---|
| gpt | 2 |
"""
)
out = realign_markdown_tables(src)
# Each table block individually aligns.
blocks: list[list[str]] = []
current: list[str] = []
for line in out.split("\n"):
if "|" in line:
current.append(line)
elif current:
blocks.append(current)
current = []
if current:
blocks.append(current)
assert len(blocks) == 2
for block in blocks:
offsets = [_column_offsets(row) for row in block]
assert all(o == offsets[0] for o in offsets), (
f"block did not align:\n" + "\n".join(block)
)

Some files were not shown because too many files have changed in this diff Show More