Initial import of NousResearch/hermes-agent
Deploy Site / deploy-vercel (push) Has been cancelled
Deploy Site / deploy-docs (push) Has been cancelled
Docker / shell lint / Lint Dockerfile (hadolint) (push) Has been cancelled
Docker / shell lint / Lint docker/ shell scripts (shellcheck) (push) Has been cancelled
Docker Build and Publish / build-amd64 (push) Has been cancelled
Docker Build and Publish / build-arm64 (push) Has been cancelled
Lint (ruff + ty) / ruff + ty diff (push) Has been cancelled
Lint (ruff + ty) / ruff enforcement (blocking) (push) Has been cancelled
Lint (ruff + ty) / Windows footguns (blocking) (push) Has been cancelled
Nix Lockfile Fix / auto-fix-main (push) Has been cancelled
Nix Lockfile Fix / fix (push) Has been cancelled
Nix / nix (macos-latest) (push) Has been cancelled
Nix / nix (ubuntu-latest) (push) Has been cancelled
OSV-Scanner / Scan lockfiles (push) Has been cancelled
Build Skills Index / build-index (push) Has been cancelled
Tests / test (1) (push) Has been cancelled
Tests / test (2) (push) Has been cancelled
Tests / test (3) (push) Has been cancelled
Tests / test (4) (push) Has been cancelled
Tests / test (5) (push) Has been cancelled
Tests / test (6) (push) Has been cancelled
Tests / e2e (push) Has been cancelled
uv.lock check / uv lock --check (push) Has been cancelled
Docker Build and Publish / merge (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
Tests / save-durations (push) Has been cancelled
Deploy Site / deploy-vercel (push) Has been cancelled
Deploy Site / deploy-docs (push) Has been cancelled
Docker / shell lint / Lint Dockerfile (hadolint) (push) Has been cancelled
Docker / shell lint / Lint docker/ shell scripts (shellcheck) (push) Has been cancelled
Docker Build and Publish / build-amd64 (push) Has been cancelled
Docker Build and Publish / build-arm64 (push) Has been cancelled
Lint (ruff + ty) / ruff + ty diff (push) Has been cancelled
Lint (ruff + ty) / ruff enforcement (blocking) (push) Has been cancelled
Lint (ruff + ty) / Windows footguns (blocking) (push) Has been cancelled
Nix Lockfile Fix / auto-fix-main (push) Has been cancelled
Nix Lockfile Fix / fix (push) Has been cancelled
Nix / nix (macos-latest) (push) Has been cancelled
Nix / nix (ubuntu-latest) (push) Has been cancelled
OSV-Scanner / Scan lockfiles (push) Has been cancelled
Build Skills Index / build-index (push) Has been cancelled
Tests / test (1) (push) Has been cancelled
Tests / test (2) (push) Has been cancelled
Tests / test (3) (push) Has been cancelled
Tests / test (4) (push) Has been cancelled
Tests / test (5) (push) Has been cancelled
Tests / test (6) (push) Has been cancelled
Tests / e2e (push) Has been cancelled
uv.lock check / uv lock --check (push) Has been cancelled
Docker Build and Publish / merge (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
Tests / save-durations (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Pytest helpers for LSP-related tests."""
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A minimal in-process LSP server used by tests.
|
||||
|
||||
Speaks just enough LSP to drive :class:`agent.lsp.client.LSPClient`
|
||||
through a full lifecycle: ``initialize``, ``initialized``,
|
||||
``textDocument/didOpen``, ``textDocument/didChange``, then a
|
||||
``textDocument/publishDiagnostics`` notification followed by
|
||||
``shutdown`` + ``exit``.
|
||||
|
||||
Behaviour (all behaviours selectable via env var ``MOCK_LSP_SCRIPT``):
|
||||
|
||||
- ``"clean"`` — initialize, accept didOpen/didChange, push empty
|
||||
diagnostics on every open/change, exit cleanly on shutdown.
|
||||
- ``"errors"`` — same as ``clean`` but the published diagnostics
|
||||
carry one severity-1 entry pointing at line 0:0.
|
||||
- ``"crash"`` — exit immediately after responding to ``initialize``
|
||||
(simulates a crashing server).
|
||||
- ``"slow"`` — same as ``clean`` but sleeps 1s before responding to
|
||||
``initialize`` (lets us test timeout behaviour).
|
||||
|
||||
The script writes JSON-RPC framed messages to stdout and reads from
|
||||
stdin. No third-party dependencies — uses only stdlib so it runs
|
||||
under whatever Python the test process picks up.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def read_message():
|
||||
"""Read one Content-Length framed JSON-RPC message from stdin."""
|
||||
headers = {}
|
||||
while True:
|
||||
line = sys.stdin.buffer.readline()
|
||||
if not line:
|
||||
return None
|
||||
line = line.rstrip(b"\r\n")
|
||||
if not line:
|
||||
break
|
||||
k, _, v = line.decode("ascii").partition(":")
|
||||
headers[k.strip().lower()] = v.strip()
|
||||
n = int(headers["content-length"])
|
||||
body = sys.stdin.buffer.read(n)
|
||||
return json.loads(body.decode("utf-8"))
|
||||
|
||||
|
||||
def write_message(obj):
|
||||
body = json.dumps(obj, separators=(",", ":")).encode("utf-8")
|
||||
sys.stdout.buffer.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii"))
|
||||
sys.stdout.buffer.write(body)
|
||||
sys.stdout.buffer.flush()
|
||||
|
||||
|
||||
def main():
|
||||
script = os.environ.get("MOCK_LSP_SCRIPT", "clean")
|
||||
|
||||
while True:
|
||||
msg = read_message()
|
||||
if msg is None:
|
||||
return 0
|
||||
|
||||
if "id" in msg and msg.get("method") == "initialize":
|
||||
if script == "slow":
|
||||
time.sleep(1.0)
|
||||
write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg["id"],
|
||||
"result": {
|
||||
"capabilities": {
|
||||
"textDocumentSync": 1, # Full
|
||||
"diagnosticProvider": {"interFileDependencies": False, "workspaceDiagnostics": False},
|
||||
},
|
||||
"serverInfo": {"name": "mock-lsp", "version": "0.1"},
|
||||
},
|
||||
}
|
||||
)
|
||||
if script == "crash":
|
||||
return 0
|
||||
continue
|
||||
|
||||
if msg.get("method") == "initialized":
|
||||
continue
|
||||
|
||||
if msg.get("method") == "workspace/didChangeConfiguration":
|
||||
continue
|
||||
|
||||
if msg.get("method") == "workspace/didChangeWatchedFiles":
|
||||
continue
|
||||
|
||||
if msg.get("method") in {"textDocument/didOpen", "textDocument/didChange"}:
|
||||
params = msg.get("params") or {}
|
||||
td = params.get("textDocument") or {}
|
||||
uri = td.get("uri", "")
|
||||
version = td.get("version", 0)
|
||||
diagnostics = []
|
||||
if script == "errors":
|
||||
diagnostics = [
|
||||
{
|
||||
"range": {
|
||||
"start": {"line": 0, "character": 0},
|
||||
"end": {"line": 0, "character": 5},
|
||||
},
|
||||
"severity": 1,
|
||||
"code": "MOCK001",
|
||||
"source": "mock-lsp",
|
||||
"message": "synthetic error from mock-lsp",
|
||||
}
|
||||
]
|
||||
write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "textDocument/publishDiagnostics",
|
||||
"params": {
|
||||
"uri": uri,
|
||||
"version": version,
|
||||
"diagnostics": diagnostics,
|
||||
},
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if msg.get("method") == "textDocument/diagnostic":
|
||||
# Pull endpoint — return empty.
|
||||
write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg["id"],
|
||||
"result": {"kind": "full", "items": []},
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if msg.get("method") == "textDocument/didSave":
|
||||
continue
|
||||
|
||||
if msg.get("method") == "shutdown":
|
||||
write_message({"jsonrpc": "2.0", "id": msg["id"], "result": None})
|
||||
continue
|
||||
|
||||
if msg.get("method") == "exit":
|
||||
return 0
|
||||
|
||||
# Unknown request: respond with method-not-found.
|
||||
if "id" in msg:
|
||||
write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg["id"],
|
||||
"error": {"code": -32601, "message": f"method not found: {msg.get('method')}"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Integration test: LSP layer is skipped on non-local backends.
|
||||
|
||||
The host-side LSP server can't see files inside a Docker/Modal/SSH
|
||||
sandbox. When the agent's terminal env isn't ``LocalEnvironment``,
|
||||
the file_operations layer must skip both ``snapshot_baseline`` and
|
||||
``get_diagnostics_sync`` calls — falling back to the in-process
|
||||
syntax check exactly as if LSP were disabled.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp import eventlog
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset():
|
||||
eventlog.reset_announce_caches()
|
||||
|
||||
|
||||
def test_local_only_helper_returns_true_for_local_env():
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
fops = ShellFileOperations(LocalEnvironment(cwd="/tmp"))
|
||||
assert fops._lsp_local_only() is True
|
||||
|
||||
|
||||
def test_local_only_helper_returns_false_for_non_local_env():
|
||||
"""A mocked non-local env (Docker/Modal/SSH stand-in) returns False."""
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
# Build something that's NOT a LocalEnvironment. We use a bare
|
||||
# MagicMock — isinstance() against LocalEnvironment is False.
|
||||
fake_env = MagicMock()
|
||||
fake_env.execute = MagicMock(return_value=MagicMock(exit_code=0, stdout=""))
|
||||
fake_env.cwd = "/sandbox"
|
||||
fops = ShellFileOperations(fake_env)
|
||||
assert fops._lsp_local_only() is False
|
||||
|
||||
|
||||
def test_snapshot_baseline_skipped_for_non_local(monkeypatch):
|
||||
"""Verify the LSP service's snapshot_baseline is NOT called when
|
||||
the backend isn't local."""
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
fake_env = MagicMock()
|
||||
fake_env.execute = MagicMock(return_value=MagicMock(exit_code=0, stdout=""))
|
||||
fake_env.cwd = "/sandbox"
|
||||
fops = ShellFileOperations(fake_env)
|
||||
|
||||
snapshot_called = []
|
||||
|
||||
class FakeService:
|
||||
def snapshot_baseline(self, path):
|
||||
snapshot_called.append(path)
|
||||
|
||||
monkeypatch.setattr("agent.lsp.get_service", lambda: FakeService())
|
||||
|
||||
fops._snapshot_lsp_baseline("/sandbox/x.py")
|
||||
assert snapshot_called == [], "snapshot must be skipped for non-local backends"
|
||||
|
||||
|
||||
def test_maybe_lsp_diagnostics_returns_empty_for_non_local(monkeypatch):
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
fake_env = MagicMock()
|
||||
fake_env.execute = MagicMock(return_value=MagicMock(exit_code=0, stdout=""))
|
||||
fake_env.cwd = "/sandbox"
|
||||
fops = ShellFileOperations(fake_env)
|
||||
|
||||
called = []
|
||||
|
||||
class FakeService:
|
||||
def enabled_for(self, path):
|
||||
called.append(("enabled_for", path))
|
||||
return True
|
||||
def get_diagnostics_sync(self, path, **kw):
|
||||
called.append(("get_diagnostics_sync", path))
|
||||
return [{"severity": 1, "message": "should not see this"}]
|
||||
|
||||
monkeypatch.setattr("agent.lsp.get_service", lambda: FakeService())
|
||||
|
||||
result = fops._maybe_lsp_diagnostics("/sandbox/x.py")
|
||||
assert result == ""
|
||||
assert called == [], "service must not be queried for non-local backends"
|
||||
|
||||
|
||||
def test_snapshot_baseline_called_for_local_env(tmp_path, monkeypatch):
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
|
||||
|
||||
snapshot_called = []
|
||||
|
||||
class FakeService:
|
||||
def snapshot_baseline(self, path):
|
||||
snapshot_called.append(path)
|
||||
|
||||
monkeypatch.setattr("agent.lsp.get_service", lambda: FakeService())
|
||||
|
||||
fops._snapshot_lsp_baseline(str(tmp_path / "x.py"))
|
||||
assert snapshot_called == [str(tmp_path / "x.py")]
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Tests for the broken-set short-circuit added to handle outer-timeout failures.
|
||||
|
||||
When ``snapshot_baseline`` or ``get_diagnostics_sync`` time out from the
|
||||
service layer (because a language server hangs during initialize, or
|
||||
the binary is wedged), the inner spawn task is cancelled — but the
|
||||
inner exception handler that adds to ``_broken`` never runs. Without
|
||||
the service-layer fallback added in this module, every subsequent
|
||||
edit re-pays the full timeout cost until the process exits.
|
||||
|
||||
This module verifies:
|
||||
- ``_mark_broken_for_file`` adds the right key
|
||||
- ``enabled_for`` short-circuits on broken keys
|
||||
- a missing binary is broken-set'd after one snapshot attempt
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp.manager import LSPService
|
||||
from agent.lsp.workspace import clear_cache
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_workspace_cache():
|
||||
clear_cache()
|
||||
yield
|
||||
clear_cache()
|
||||
|
||||
|
||||
def _make_git_workspace(tmp_path: Path) -> Path:
|
||||
"""Build a minimal git repo with a pyproject so pyright's root resolver fires."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / ".git").mkdir()
|
||||
(repo / "pyproject.toml").write_text("[project]\nname='t'\n")
|
||||
return repo
|
||||
|
||||
|
||||
def test_mark_broken_for_file_adds_correct_key(tmp_path, monkeypatch):
|
||||
"""``_mark_broken_for_file`` keys the broken-set on
|
||||
(server_id, per_server_root) so subsequent ``enabled_for`` calls
|
||||
for files in the same project skip immediately."""
|
||||
repo = _make_git_workspace(tmp_path)
|
||||
monkeypatch.chdir(str(repo))
|
||||
src = repo / "x.py"
|
||||
src.write_text("")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc._mark_broken_for_file(str(src), RuntimeError("simulated"))
|
||||
# The pyright server resolves to the repo root via pyproject.toml.
|
||||
assert ("pyright", str(repo)) in svc._broken
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_enabled_for_returns_false_after_broken(tmp_path, monkeypatch):
|
||||
"""Once a (server_id, root) pair is in the broken-set,
|
||||
``enabled_for`` returns False so the file_operations layer skips
|
||||
the LSP path entirely."""
|
||||
repo = _make_git_workspace(tmp_path)
|
||||
monkeypatch.chdir(str(repo))
|
||||
src = repo / "x.py"
|
||||
src.write_text("")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
# Initially enabled.
|
||||
assert svc.enabled_for(str(src)) is True
|
||||
# Mark broken.
|
||||
svc._mark_broken_for_file(str(src), RuntimeError("simulated"))
|
||||
# Now disabled — the broken-set short-circuits.
|
||||
assert svc.enabled_for(str(src)) is False
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_enabled_for_other_file_in_same_project_also_skipped(tmp_path, monkeypatch):
|
||||
"""The broken key is (server_id, root), so ALL files routed through
|
||||
the same server in the same project are skipped — not just the one
|
||||
that triggered the failure."""
|
||||
repo = _make_git_workspace(tmp_path)
|
||||
monkeypatch.chdir(str(repo))
|
||||
a = repo / "a.py"
|
||||
a.write_text("")
|
||||
b = repo / "b.py"
|
||||
b.write_text("")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc._mark_broken_for_file(str(a), RuntimeError("simulated"))
|
||||
# Both files in the same project skip pyright now.
|
||||
assert svc.enabled_for(str(a)) is False
|
||||
assert svc.enabled_for(str(b)) is False
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_unrelated_project_not_affected_by_broken(tmp_path, monkeypatch):
|
||||
"""Marking pyright broken for project A must NOT affect project B."""
|
||||
repo_a = _make_git_workspace(tmp_path)
|
||||
repo_b = tmp_path / "repo-b"
|
||||
repo_b.mkdir()
|
||||
(repo_b / ".git").mkdir()
|
||||
(repo_b / "pyproject.toml").write_text("[project]\nname='b'\n")
|
||||
a_src = repo_a / "x.py"
|
||||
a_src.write_text("")
|
||||
b_src = repo_b / "x.py"
|
||||
b_src.write_text("")
|
||||
|
||||
monkeypatch.chdir(str(repo_a))
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc._mark_broken_for_file(str(a_src), RuntimeError("simulated"))
|
||||
# Project A skipped.
|
||||
assert svc.enabled_for(str(a_src)) is False
|
||||
# Project B still enabled — the broken key is per-project.
|
||||
monkeypatch.chdir(str(repo_b))
|
||||
assert svc.enabled_for(str(b_src)) is True
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_mark_broken_handles_missing_server_silently(tmp_path):
|
||||
"""If the file extension doesn't match any registered server,
|
||||
``_mark_broken_for_file`` no-ops — nothing to mark."""
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
# No registered server for .xyz; must not raise.
|
||||
svc._mark_broken_for_file(str(tmp_path / "weird.xyz"), RuntimeError("x"))
|
||||
assert len(svc._broken) == 0
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_mark_broken_handles_no_workspace_silently(tmp_path):
|
||||
"""File outside any git worktree → no workspace → no key to add."""
|
||||
src = tmp_path / "orphan.py"
|
||||
src.write_text("")
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc._mark_broken_for_file(str(src), RuntimeError("x"))
|
||||
assert len(svc._broken) == 0
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_snapshot_failure_marks_broken_via_outer_timeout(tmp_path, monkeypatch):
|
||||
"""End-to-end: ``snapshot_baseline``'s outer ``_loop.run`` timeout
|
||||
triggers ``_mark_broken_for_file``, so a second call to
|
||||
``enabled_for`` returns False."""
|
||||
repo = _make_git_workspace(tmp_path)
|
||||
monkeypatch.chdir(str(repo))
|
||||
src = repo / "x.py"
|
||||
src.write_text("")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
# Force the inner snapshot coroutine to raise.
|
||||
async def boom(_path):
|
||||
raise RuntimeError("outer-timeout simulated")
|
||||
|
||||
with patch.object(svc, "_snapshot_async", boom):
|
||||
assert svc.enabled_for(str(src)) is True
|
||||
svc.snapshot_baseline(str(src))
|
||||
|
||||
# After the failure, the file's pair is in the broken-set and
|
||||
# ``enabled_for`` skips it.
|
||||
assert ("pyright", str(repo)) in svc._broken
|
||||
assert svc.enabled_for(str(src)) is False
|
||||
finally:
|
||||
svc.shutdown()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""End-to-end client tests against the in-process mock LSP server.
|
||||
|
||||
Spins up :file:`_mock_lsp_server.py` as an actual subprocess, drives
|
||||
it through real LSP traffic, and asserts diagnostic flow. This is
|
||||
the closest thing we have to integration coverage without requiring
|
||||
pyright/gopls/etc. to be installed in CI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp.client import LSPClient
|
||||
|
||||
|
||||
MOCK_SERVER = str(Path(__file__).parent / "_mock_lsp_server.py")
|
||||
|
||||
|
||||
def _client(workspace: Path, script: str = "clean") -> LSPClient:
|
||||
env = {"MOCK_LSP_SCRIPT": script, "PYTHONPATH": os.environ.get("PYTHONPATH", "")}
|
||||
return LSPClient(
|
||||
server_id=f"mock-{script}",
|
||||
workspace_root=str(workspace),
|
||||
command=[sys.executable, MOCK_SERVER],
|
||||
env=env,
|
||||
cwd=str(workspace),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_lifecycle_clean(tmp_path: Path):
|
||||
"""Full lifecycle: spawn, initialize, open, get clean diagnostics, shutdown."""
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
client = _client(tmp_path, "clean")
|
||||
await client.start()
|
||||
try:
|
||||
assert client.is_running
|
||||
version = await client.open_file(str(f), language_id="python")
|
||||
assert version == 0
|
||||
await client.wait_for_diagnostics(str(f), version, mode="document")
|
||||
diags = client.diagnostics_for(str(f))
|
||||
assert diags == []
|
||||
finally:
|
||||
await client.shutdown()
|
||||
assert not client.is_running
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_receives_published_errors(tmp_path: Path):
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
client = _client(tmp_path, "errors")
|
||||
await client.start()
|
||||
try:
|
||||
version = await client.open_file(str(f), language_id="python")
|
||||
await client.wait_for_diagnostics(str(f), version, mode="document")
|
||||
diags = client.diagnostics_for(str(f))
|
||||
assert len(diags) == 1
|
||||
d = diags[0]
|
||||
assert d["severity"] == 1
|
||||
assert d["code"] == "MOCK001"
|
||||
assert d["source"] == "mock-lsp"
|
||||
assert "synthetic error" in d["message"]
|
||||
finally:
|
||||
await client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_didchange_bumps_version(tmp_path: Path):
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
client = _client(tmp_path, "errors")
|
||||
await client.start()
|
||||
try:
|
||||
v0 = await client.open_file(str(f), language_id="python")
|
||||
f.write_text("print('hi 2')\n")
|
||||
v1 = await client.open_file(str(f), language_id="python") # re-open path = didChange
|
||||
assert v1 == v0 + 1
|
||||
await client.wait_for_diagnostics(str(f), v1, mode="document")
|
||||
# Mock pushed a diagnostic for both events; merged view has one
|
||||
# entry (push store keyed by file path).
|
||||
diags = client.diagnostics_for(str(f))
|
||||
assert len(diags) == 1
|
||||
finally:
|
||||
await client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_handles_crashing_server(tmp_path: Path):
|
||||
"""When the server exits right after initialize, subsequent requests
|
||||
fail gracefully (not hang)."""
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
|
||||
client = _client(tmp_path, "crash")
|
||||
await client.start() # should succeed (mock answers initialize before crashing)
|
||||
# Give the OS a moment to deliver the EOF.
|
||||
await asyncio.sleep(0.2)
|
||||
# The reader loop should detect EOF and mark pending requests as failed.
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
client.open_file(str(f), language_id="python"), timeout=2.0
|
||||
)
|
||||
except Exception:
|
||||
pass # any exception is acceptable; the contract is "doesn't hang"
|
||||
await client.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_shutdown_idempotent(tmp_path: Path):
|
||||
"""Calling shutdown twice must be safe."""
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
client = _client(tmp_path, "clean")
|
||||
await client.start()
|
||||
await client.shutdown()
|
||||
await client.shutdown() # must not raise
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_diagnostics_are_deduped(tmp_path: Path):
|
||||
"""Repeated identical pushes must not produce duplicate diagnostics."""
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
client = _client(tmp_path, "errors")
|
||||
await client.start()
|
||||
try:
|
||||
for _ in range(3):
|
||||
v = await client.open_file(str(f), language_id="python")
|
||||
await client.wait_for_diagnostics(str(f), v, mode="document")
|
||||
diags = client.diagnostics_for(str(f))
|
||||
# Push store overwrites on every notification — should have 1.
|
||||
assert len(diags) == 1
|
||||
finally:
|
||||
await client.shutdown()
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Tests for cross-edit LSP delta filtering.
|
||||
|
||||
The delta-filter contract spans three pieces:
|
||||
|
||||
1. ``agent.lsp.manager._diag_key`` — strict equality key including
|
||||
the diagnostic's position range. Two diagnostics with the same
|
||||
content but different lines are NOT equal under this key (they
|
||||
are genuinely different diagnostics).
|
||||
2. ``agent.lsp.range_shift.build_line_shift`` — derives a function
|
||||
mapping pre-edit line numbers to post-edit line numbers from a
|
||||
pre/post text pair.
|
||||
3. ``agent.lsp.manager.LSPService.get_diagnostics_sync(line_shift=…)``
|
||||
— applies the shift to baseline diagnostics before computing the
|
||||
set-difference, so pre-existing errors at shifted lines hash
|
||||
equal to their post-edit counterparts and get filtered out.
|
||||
|
||||
These tests exercise the contract at the unit level; the E2E case
|
||||
(real LSP server, real shift) is covered in test_service.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from agent.lsp.client import _diagnostic_key
|
||||
from agent.lsp.manager import _diag_key
|
||||
from agent.lsp.range_shift import (
|
||||
build_line_shift,
|
||||
shift_baseline,
|
||||
shift_diagnostic_range,
|
||||
)
|
||||
|
||||
|
||||
def _diag(*, line: int, message: str = "Undefined variable",
|
||||
severity: int = 1, code: str = "reportUndefinedVariable",
|
||||
source: str = "Pyright", end_line: int | None = None) -> dict:
|
||||
if end_line is None:
|
||||
end_line = line
|
||||
return {
|
||||
"severity": severity,
|
||||
"code": code,
|
||||
"source": source,
|
||||
"message": message,
|
||||
"range": {
|
||||
"start": {"line": line, "character": 0},
|
||||
"end": {"line": end_line, "character": 10},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# _diag_key: strict equality (with range)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_diag_key_treats_shifted_diagnostics_as_distinct():
|
||||
"""Two diagnostics with the same message but at different lines hash
|
||||
differently — they are genuinely different diagnostics. The shift
|
||||
map is what makes them equal AFTER remapping; the key itself stays
|
||||
strict."""
|
||||
a = _diag(line=100)
|
||||
b = _diag(line=200)
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_matches_client_key_for_shifted_baseline():
|
||||
"""When a baseline diagnostic is remapped through a shift, its
|
||||
_diag_key must match the corresponding post-edit diagnostic's key
|
||||
at the same coordinates. This is the contract the delta filter
|
||||
relies on."""
|
||||
pre = _diag(line=200)
|
||||
# Edit deletes 14 lines above line 200, so the same error now
|
||||
# appears at line 186 post-edit.
|
||||
shift = lambda L: L - 14 if L >= 14 else L
|
||||
shifted = shift_diagnostic_range(pre, shift)
|
||||
assert shifted is not None
|
||||
post = _diag(line=186)
|
||||
assert _diag_key(shifted) == _diag_key(post)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_message():
|
||||
a = _diag(line=100, message="foo")
|
||||
b = _diag(line=100, message="bar")
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_severity():
|
||||
a = _diag(line=100, severity=1)
|
||||
b = _diag(line=100, severity=2)
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_source():
|
||||
a = _diag(line=100, source="Pyright")
|
||||
b = _diag(line=100, source="Ruff")
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_matches_client_key_byte_for_byte():
|
||||
"""The manager-side and client-side keys must agree on diagnostic
|
||||
identity — they're used by two layers that need to round-trip the
|
||||
same diagnostics through dedup and delta filtering."""
|
||||
d = _diag(line=42)
|
||||
assert _diag_key(d) == _diagnostic_key(d)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# build_line_shift
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_shift_identity_for_identical_content():
|
||||
shift = build_line_shift("a\nb\nc\n", "a\nb\nc\n")
|
||||
assert shift(0) == 0
|
||||
assert shift(1) == 1
|
||||
assert shift(2) == 2
|
||||
|
||||
|
||||
def test_shift_pure_deletion_above_line():
|
||||
"""Delete 2 lines at the top; everything below shifts up by 2."""
|
||||
pre = "line0\nline1\nline2\nline3\nline4\n"
|
||||
post = "line2\nline3\nline4\n" # deleted lines 0-1
|
||||
shift = build_line_shift(pre, post)
|
||||
# Pre lines 0,1 → deleted → None
|
||||
assert shift(0) is None
|
||||
assert shift(1) is None
|
||||
# Pre line 2 → post line 0
|
||||
assert shift(2) == 0
|
||||
# Pre line 4 → post line 2
|
||||
assert shift(4) == 2
|
||||
|
||||
|
||||
def test_shift_pure_insertion_above_line():
|
||||
"""Insert 3 lines at the top; everything below shifts down by 3."""
|
||||
pre = "line0\nline1\nline2\n"
|
||||
post = "new0\nnew1\nnew2\nline0\nline1\nline2\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
# Pre lines unchanged in identity, shifted by 3
|
||||
assert shift(0) == 3
|
||||
assert shift(1) == 4
|
||||
assert shift(2) == 5
|
||||
|
||||
|
||||
def test_shift_replacement_in_middle():
|
||||
"""Replace 2 lines in the middle with 1 line. Lines above
|
||||
unchanged; lines below shift up by 1."""
|
||||
pre = "a\nb\nc\nd\ne\n"
|
||||
post = "a\nb\nX\ne\n" # replaced lines 2,3 (c,d) with X
|
||||
shift = build_line_shift(pre, post)
|
||||
assert shift(0) == 0 # a → a
|
||||
assert shift(1) == 1 # b → b
|
||||
assert shift(2) is None # c → deleted
|
||||
assert shift(3) is None # d → deleted
|
||||
assert shift(4) == 3 # e → post line 3
|
||||
|
||||
|
||||
def test_shift_handles_empty_pre():
|
||||
"""First write of a file: pre is empty, post has content. Nothing
|
||||
to shift, so the function should be well-defined for empty pre."""
|
||||
shift = build_line_shift("", "hello\nworld\n")
|
||||
# Any pre line falls past the end of an empty pre — anchor at end of post
|
||||
assert shift(0) == 1
|
||||
|
||||
|
||||
def test_shift_handles_empty_post():
|
||||
"""File deleted to empty. Every pre line returns None."""
|
||||
shift = build_line_shift("line0\nline1\n", "")
|
||||
assert shift(0) is None
|
||||
assert shift(1) is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# shift_diagnostic_range
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_shift_diag_remaps_start_and_end():
|
||||
pre = "a\nb\nc\nd\n"
|
||||
post = "X\na\nb\nc\nd\n" # one line inserted at top
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=2, end_line=2)
|
||||
remapped = shift_diagnostic_range(d, shift)
|
||||
assert remapped is not None
|
||||
assert remapped["range"]["start"]["line"] == 3
|
||||
assert remapped["range"]["end"]["line"] == 3
|
||||
|
||||
|
||||
def test_shift_diag_drops_diagnostic_in_deleted_region():
|
||||
pre = "a\nb\nc\nd\n"
|
||||
post = "a\nd\n" # deleted lines 1,2 (b,c)
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=1)
|
||||
assert shift_diagnostic_range(d, shift) is None
|
||||
|
||||
|
||||
def test_shift_diag_does_not_mutate_original():
|
||||
pre = "a\nb\n"
|
||||
post = "X\na\nb\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=0)
|
||||
original_line = d["range"]["start"]["line"]
|
||||
_ = shift_diagnostic_range(d, shift)
|
||||
assert d["range"]["start"]["line"] == original_line
|
||||
|
||||
|
||||
def test_shift_baseline_drops_deleted_and_remaps_rest():
|
||||
pre = "a\nb\nc\nd\ne\n"
|
||||
post = "a\ne\n" # deleted b,c,d
|
||||
shift = build_line_shift(pre, post)
|
||||
baseline = [
|
||||
_diag(line=0, message="err on a"),
|
||||
_diag(line=1, message="err on b"), # → deleted
|
||||
_diag(line=2, message="err on c"), # → deleted
|
||||
_diag(line=4, message="err on e"),
|
||||
]
|
||||
out = shift_baseline(baseline, shift)
|
||||
assert [d["message"] for d in out] == ["err on a", "err on e"]
|
||||
assert out[0]["range"]["start"]["line"] == 0
|
||||
assert out[1]["range"]["start"]["line"] == 1
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# End-to-end: simulate the delta-filter pipeline
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_pipeline_filters_shifted_baseline_under_strict_key():
|
||||
"""The exact scenario the bug fix is for: an edit deletes lines,
|
||||
every diagnostic below shifts, and the delta filter (strict key
|
||||
+ shifted baseline) correctly identifies them as pre-existing."""
|
||||
pre = "line0\nline1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\n"
|
||||
# Delete lines 2,3,4 — pre-existing errors at lines 7,8 should
|
||||
# appear at lines 4,5 post-edit and be filtered out.
|
||||
post = "line0\nline1\nline5\nline6\nline7\nline8\nline9\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
|
||||
baseline = [_diag(line=7, message="X"), _diag(line=8, message="Y")]
|
||||
post_diags = [_diag(line=4, message="X"), _diag(line=5, message="Y")]
|
||||
|
||||
shifted_baseline = shift_baseline(baseline, shift)
|
||||
seen = {_diag_key(d) for d in shifted_baseline}
|
||||
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
|
||||
|
||||
# Both errors were pre-existing — filtered out.
|
||||
assert new_diags == []
|
||||
|
||||
|
||||
def test_pipeline_preserves_new_instance_at_different_line():
|
||||
"""The case content-only keys would miss: the model introduces a
|
||||
SECOND instance of the same error class at a new location. The
|
||||
new instance must surface."""
|
||||
pre = "good\ngood\ngood\n"
|
||||
post = "good\nbad\ngood\nbad\n" # added 2 new error lines
|
||||
shift = build_line_shift(pre, post)
|
||||
|
||||
baseline = [_diag(line=0, message="bad style")] # pre-existing
|
||||
post_diags = [
|
||||
_diag(line=0, message="bad style"), # pre-existing
|
||||
_diag(line=1, message="bad style"), # NEW — different line
|
||||
_diag(line=3, message="bad style"), # NEW — different line
|
||||
]
|
||||
|
||||
shifted_baseline = shift_baseline(baseline, shift)
|
||||
seen = {_diag_key(d) for d in shifted_baseline}
|
||||
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
|
||||
|
||||
# Two genuinely new instances must be surfaced.
|
||||
assert len(new_diags) == 2
|
||||
assert {d["range"]["start"]["line"] for d in new_diags} == {1, 3}
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Tests for the ``lsp_diagnostics`` field on WriteResult / PatchResult.
|
||||
|
||||
The field exists so the agent can read syntax errors (``lint``) and
|
||||
semantic errors (``lsp_diagnostics``) as separate signals rather than
|
||||
having LSP output prepended to the lint string.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.file_operations import (
|
||||
PatchResult,
|
||||
ShellFileOperations,
|
||||
WriteResult,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclass shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_writeresult_lsp_diagnostics_optional():
|
||||
r = WriteResult()
|
||||
assert r.lsp_diagnostics is None
|
||||
|
||||
|
||||
def test_writeresult_to_dict_omits_field_when_none():
|
||||
r = WriteResult(bytes_written=10)
|
||||
assert "lsp_diagnostics" not in r.to_dict()
|
||||
|
||||
|
||||
def test_writeresult_to_dict_includes_field_when_set():
|
||||
r = WriteResult(bytes_written=10, lsp_diagnostics="<diagnostics>...</diagnostics>")
|
||||
d = r.to_dict()
|
||||
assert d["lsp_diagnostics"] == "<diagnostics>...</diagnostics>"
|
||||
|
||||
|
||||
def test_patchresult_to_dict_includes_field_when_set():
|
||||
r = PatchResult(success=True, lsp_diagnostics="ERROR [1:1] thing")
|
||||
d = r.to_dict()
|
||||
assert d["lsp_diagnostics"] == "ERROR [1:1] thing"
|
||||
|
||||
|
||||
def test_patchresult_to_dict_omits_field_when_none():
|
||||
r = PatchResult(success=True)
|
||||
assert "lsp_diagnostics" not in r.to_dict()
|
||||
|
||||
|
||||
def test_patchresult_to_dict_omits_field_when_empty_string():
|
||||
"""Empty string counts as falsy — agent shouldn't see an empty field."""
|
||||
r = PatchResult(success=True, lsp_diagnostics="")
|
||||
assert "lsp_diagnostics" not in r.to_dict()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Channel separation: lint and lsp_diagnostics stay independent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_lint_and_lsp_diagnostics_are_separate_channels():
|
||||
"""A WriteResult can carry BOTH a syntax-error lint AND an LSP
|
||||
diagnostic block. They belong in separate fields."""
|
||||
r = WriteResult(
|
||||
bytes_written=42,
|
||||
lint={"status": "error", "output": "SyntaxError: ..."},
|
||||
lsp_diagnostics="<diagnostics>ERROR [1:5] type mismatch</diagnostics>",
|
||||
)
|
||||
d = r.to_dict()
|
||||
assert "lint" in d
|
||||
assert "lsp_diagnostics" in d
|
||||
assert d["lint"]["output"] == "SyntaxError: ..."
|
||||
assert "type mismatch" in d["lsp_diagnostics"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# write_file populates the field via _maybe_lsp_diagnostics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_file_populates_lsp_diagnostics_when_layer_returns_block(tmp_path):
|
||||
"""When the LSP layer returns a non-empty block, write_file puts it
|
||||
into the ``lsp_diagnostics`` field — NOT into ``lint.output``."""
|
||||
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
|
||||
target = tmp_path / "x.py"
|
||||
|
||||
block = "<diagnostics file=\"x.py\">\nERROR [1:1] problem\n</diagnostics>"
|
||||
|
||||
with patch.object(fops, "_maybe_lsp_diagnostics", return_value=block):
|
||||
res = fops.write_file(str(target), "x = 1\n")
|
||||
|
||||
assert res.lsp_diagnostics == block
|
||||
# Lint is the syntax check, which is clean for "x = 1" — must NOT
|
||||
# have the LSP block folded into it.
|
||||
assert res.lint == {"status": "ok", "output": ""}
|
||||
|
||||
|
||||
def test_write_file_lsp_diagnostics_none_when_layer_returns_empty(tmp_path):
|
||||
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
|
||||
target = tmp_path / "x.py"
|
||||
|
||||
with patch.object(fops, "_maybe_lsp_diagnostics", return_value=""):
|
||||
res = fops.write_file(str(target), "x = 1\n")
|
||||
|
||||
assert res.lsp_diagnostics is None
|
||||
|
||||
|
||||
def test_write_file_skips_lsp_when_syntax_failed(tmp_path):
|
||||
"""If the syntax check finds errors, the LSP layer should not be
|
||||
consulted (a file that won't parse won't yield meaningful semantic
|
||||
diagnostics)."""
|
||||
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
|
||||
target = tmp_path / "broken.py"
|
||||
|
||||
with patch.object(fops, "_maybe_lsp_diagnostics") as mock_lsp:
|
||||
res = fops.write_file(str(target), "def x(:\n") # syntax error
|
||||
assert mock_lsp.call_count == 0
|
||||
assert res.lsp_diagnostics is None
|
||||
assert res.lint["status"] == "error"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# patch_replace propagates the field from the inner write_file
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_patch_replace_propagates_lsp_diagnostics(tmp_path):
|
||||
"""patch_replace's internal write_file populates lsp_diagnostics —
|
||||
the outer PatchResult must carry it forward."""
|
||||
fops = ShellFileOperations(LocalEnvironment(cwd=str(tmp_path)))
|
||||
target = tmp_path / "x.py"
|
||||
target.write_text("x = 1\n")
|
||||
|
||||
block = "<diagnostics>ERROR [1:5] semantic issue</diagnostics>"
|
||||
|
||||
with patch.object(fops, "_maybe_lsp_diagnostics", return_value=block):
|
||||
res = fops.patch_replace(str(target), "x = 1", "x = 2")
|
||||
|
||||
assert res.success is True
|
||||
assert res.lsp_diagnostics == block
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Tests for the structured logging dedup model.
|
||||
|
||||
The contract: a 1000-write session in one project should emit exactly
|
||||
ONE INFO line ("active for <root>") at the default INFO threshold.
|
||||
Steady-state events stay at DEBUG; first-time-seen events surface
|
||||
once at INFO/WARNING.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp import eventlog
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset():
|
||||
eventlog.reset_announce_caches()
|
||||
yield
|
||||
eventlog.reset_announce_caches()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def caplog_lsp(caplog):
|
||||
caplog.set_level(logging.DEBUG, logger="hermes.lint.lsp")
|
||||
return caplog
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Steady-state silence (DEBUG)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_clean_emits_at_debug(caplog_lsp):
|
||||
for _ in range(10):
|
||||
eventlog.log_clean("pyright", "/proj/x.py")
|
||||
info_records = [r for r in caplog_lsp.records if r.levelno >= logging.INFO]
|
||||
debug_records = [r for r in caplog_lsp.records if r.levelno == logging.DEBUG]
|
||||
assert info_records == []
|
||||
assert len(debug_records) == 10
|
||||
|
||||
|
||||
def test_disabled_emits_at_debug(caplog_lsp):
|
||||
eventlog.log_disabled("pyright", "/x.py", "feature off")
|
||||
eventlog.log_disabled("pyright", "/x.py", "ext not mapped")
|
||||
assert all(r.levelno == logging.DEBUG for r in caplog_lsp.records)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State transitions: INFO once, DEBUG thereafter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_active_for_fires_once_per_root(caplog_lsp):
|
||||
for _ in range(50):
|
||||
eventlog.log_active("pyright", "/proj")
|
||||
info_records = [
|
||||
r for r in caplog_lsp.records
|
||||
if r.levelno == logging.INFO and "active for" in r.getMessage()
|
||||
]
|
||||
assert len(info_records) == 1
|
||||
|
||||
|
||||
def test_active_for_fires_per_distinct_root(caplog_lsp):
|
||||
eventlog.log_active("pyright", "/proj-a")
|
||||
eventlog.log_active("pyright", "/proj-b")
|
||||
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
|
||||
assert len(info) == 2
|
||||
|
||||
|
||||
def test_active_for_separate_per_server(caplog_lsp):
|
||||
eventlog.log_active("pyright", "/proj")
|
||||
eventlog.log_active("typescript", "/proj")
|
||||
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
|
||||
assert len(info) == 2
|
||||
|
||||
|
||||
def test_no_project_root_fires_once_per_path(caplog_lsp):
|
||||
for _ in range(5):
|
||||
eventlog.log_no_project_root("pyright", "/orphan.py")
|
||||
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
|
||||
assert len(info) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Diagnostics events fire INFO every time
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_diagnostics_always_info(caplog_lsp):
|
||||
for i in range(5):
|
||||
eventlog.log_diagnostics("pyright", f"/x{i}.py", 1)
|
||||
info = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
|
||||
assert len(info) == 5
|
||||
assert all("diags" in r.getMessage() for r in info)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action-required: WARNING once, DEBUG thereafter (or per call for novel events)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_server_unavailable_warns_once_per_binary(caplog_lsp):
|
||||
for _ in range(20):
|
||||
eventlog.log_server_unavailable("pyright", "pyright-langserver")
|
||||
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
|
||||
assert len(warns) == 1
|
||||
assert "pyright-langserver" in warns[0].getMessage()
|
||||
|
||||
|
||||
def test_server_unavailable_separate_per_binary(caplog_lsp):
|
||||
eventlog.log_server_unavailable("pyright", "pyright-langserver")
|
||||
eventlog.log_server_unavailable("typescript", "typescript-language-server")
|
||||
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
|
||||
assert len(warns) == 2
|
||||
|
||||
|
||||
def test_no_server_configured_warns_once(caplog_lsp):
|
||||
for _ in range(10):
|
||||
eventlog.log_no_server_configured("pyright")
|
||||
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
|
||||
assert len(warns) == 1
|
||||
|
||||
|
||||
def test_timeout_warns_every_call(caplog_lsp):
|
||||
for _ in range(3):
|
||||
eventlog.log_timeout("pyright", "/x.py")
|
||||
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
|
||||
assert len(warns) == 3
|
||||
|
||||
|
||||
def test_server_error_warns_every_call(caplog_lsp):
|
||||
for _ in range(3):
|
||||
eventlog.log_server_error("pyright", "/x.py", RuntimeError("boom"))
|
||||
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
|
||||
assert len(warns) == 3
|
||||
|
||||
|
||||
def test_spawn_failed_warns(caplog_lsp):
|
||||
eventlog.log_spawn_failed("pyright", "/proj", FileNotFoundError("nope"))
|
||||
warns = [r for r in caplog_lsp.records if r.levelno == logging.WARNING]
|
||||
assert len(warns) == 1
|
||||
assert "spawn/initialize failed" in warns[0].getMessage()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Format: log lines all carry the lsp[<server_id>] prefix for grep
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_log_lines_use_lsp_prefix(caplog_lsp):
|
||||
eventlog.log_clean("pyright", "/x.py")
|
||||
eventlog.log_active("pyright", "/proj")
|
||||
eventlog.log_diagnostics("typescript", "/y.ts", 2)
|
||||
for r in caplog_lsp.records:
|
||||
assert r.getMessage().startswith("lsp[")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Steady-state contract: 1000 clean writes → 1 INFO at most
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_thousand_clean_writes_emit_one_info(caplog_lsp):
|
||||
"""A long session writes lots of files cleanly; agent.log should
|
||||
show ONE 'active for' INFO and zero other INFO lines."""
|
||||
eventlog.log_active("pyright", "/proj")
|
||||
for _ in range(1000):
|
||||
eventlog.log_clean("pyright", "/proj/x.py")
|
||||
info_records = [r for r in caplog_lsp.records if r.levelno == logging.INFO]
|
||||
assert len(info_records) == 1
|
||||
assert "active for" in info_records[0].getMessage()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path shortening
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_short_path_uses_relative_when_inside_cwd(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
sub = tmp_path / "x.py"
|
||||
sub.write_text("")
|
||||
out = eventlog._short_path(str(sub))
|
||||
assert out == "x.py"
|
||||
|
||||
|
||||
def test_short_path_keeps_absolute_when_outside(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path / "a") if (tmp_path / "a").exists() else None
|
||||
monkeypatch.chdir(tmp_path)
|
||||
other = "/var/log/foo.txt"
|
||||
out = eventlog._short_path(other)
|
||||
# Outside cwd: keeps absolute (no leading "../")
|
||||
assert out == "/var/log/foo.txt" or not out.startswith("..")
|
||||
|
||||
|
||||
def test_short_path_handles_empty_string():
|
||||
assert eventlog._short_path("") == ""
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Tests for follow-up fixes to the LSP integration (PR after #24168).
|
||||
|
||||
Covers:
|
||||
|
||||
1. ``typescript-language-server`` install recipe pulls in ``typescript``
|
||||
alongside the server, so the npm install command targets both.
|
||||
2. ``hermes lsp status`` surfaces a ``Backend warnings`` section when
|
||||
bash-language-server is installed but ``shellcheck`` is missing.
|
||||
3. ``_check_lint`` returns ``skipped`` (not ``error``) when the linter
|
||||
command exists on PATH but couldn't actually run — e.g. ``npx tsc``
|
||||
without the typescript SDK installed. This is what unblocks the
|
||||
LSP semantic tier on TypeScript files when the user doesn't also
|
||||
have a project-level ``tsc``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp.install import INSTALL_RECIPES
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 1: typescript install recipe carries the typescript SDK
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_typescript_recipe_includes_typescript_sdk():
|
||||
recipe = INSTALL_RECIPES["typescript-language-server"]
|
||||
extras = recipe.get("extra_pkgs") or []
|
||||
assert "typescript" in extras, (
|
||||
"typescript-language-server requires the `typescript` SDK as a "
|
||||
"sibling install — without it `initialize` fails with "
|
||||
"'Could not find a valid TypeScript installation'."
|
||||
)
|
||||
|
||||
|
||||
def test_install_npm_passes_extras_to_npm_command(tmp_path, monkeypatch):
|
||||
"""Verify the npm subprocess is invoked with both pkg AND extras."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
captured["cmd"] = cmd
|
||||
# Pretend npm succeeded but binary doesn't exist — install code
|
||||
# will return None, which is fine for this test.
|
||||
return MagicMock(returncode=0, stderr="")
|
||||
|
||||
from agent.lsp import install as install_mod
|
||||
|
||||
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(install_mod.shutil, "which", lambda c: "/usr/bin/npm" if c == "npm" else None)
|
||||
|
||||
install_mod._install_npm("typescript-language-server", "typescript-language-server",
|
||||
extra_pkgs=["typescript"])
|
||||
|
||||
cmd = captured["cmd"]
|
||||
assert "typescript-language-server" in cmd
|
||||
assert "typescript" in cmd
|
||||
# Both must come AFTER the npm flags, in install-target position
|
||||
install_idx = cmd.index("install")
|
||||
assert cmd.index("typescript-language-server") > install_idx
|
||||
assert cmd.index("typescript") > install_idx
|
||||
|
||||
|
||||
def test_install_npm_works_without_extras(tmp_path, monkeypatch):
|
||||
"""Backwards compat: pyright-style recipes (no extras) still install."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
captured["cmd"] = cmd
|
||||
return MagicMock(returncode=0, stderr="")
|
||||
|
||||
from agent.lsp import install as install_mod
|
||||
|
||||
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(install_mod.shutil, "which", lambda c: "/usr/bin/npm" if c == "npm" else None)
|
||||
|
||||
install_mod._install_npm("pyright", "pyright-langserver")
|
||||
|
||||
cmd = captured["cmd"]
|
||||
assert "pyright" in cmd
|
||||
# Should not blow up when extra_pkgs is omitted/None
|
||||
install_targets = [c for c in cmd if not c.startswith("-") and c not in {
|
||||
"install", "--prefix", str(install_mod.hermes_lsp_bin_dir().parent),
|
||||
"/usr/bin/npm",
|
||||
}]
|
||||
assert install_targets == ["pyright"]
|
||||
|
||||
|
||||
def test_existing_binary_finds_windows_wrapper_in_staging(tmp_path, monkeypatch):
|
||||
"""Installed Windows shims should satisfy later status/probe calls."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
from agent.lsp import install as install_mod
|
||||
|
||||
wrapper = install_mod.hermes_lsp_bin_dir() / "pyright-langserver.cmd"
|
||||
wrapper.write_text("@echo off\n")
|
||||
wrapper.chmod(0o755)
|
||||
|
||||
monkeypatch.setattr(install_mod, "_is_windows", lambda: True)
|
||||
monkeypatch.setattr(install_mod.shutil, "which", lambda _name: None)
|
||||
|
||||
assert install_mod._existing_binary("pyright-langserver") == str(wrapper)
|
||||
assert install_mod.detect_status("pyright") == "installed"
|
||||
|
||||
|
||||
def test_install_pip_finds_windows_scripts_launcher(tmp_path, monkeypatch):
|
||||
"""pip console scripts can land in Scripts/ on native Windows."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
from agent.lsp import install as install_mod
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
scripts_dir = install_mod.hermes_lsp_bin_dir().parent / "python-packages" / "Scripts"
|
||||
scripts_dir.mkdir(parents=True, exist_ok=True)
|
||||
launcher = scripts_dir / "fake-language-server.exe"
|
||||
launcher.write_text("launcher\n")
|
||||
launcher.chmod(0o755)
|
||||
return MagicMock(returncode=0, stderr="")
|
||||
|
||||
monkeypatch.setattr(install_mod, "_is_windows", lambda: True)
|
||||
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
|
||||
|
||||
resolved = install_mod._install_pip("fake-lsp", "fake-language-server")
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.endswith("fake-language-server.exe")
|
||||
assert (install_mod.hermes_lsp_bin_dir() / "fake-language-server.exe").exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 2: ``hermes lsp status`` surfaces shellcheck-missing for bash
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_backend_warnings_quiet_when_bash_not_installed(tmp_path, monkeypatch):
|
||||
"""No bash → no warning."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
from agent.lsp import cli as lsp_cli
|
||||
|
||||
with patch("shutil.which", return_value=None):
|
||||
notes = lsp_cli._backend_warnings()
|
||||
assert notes == []
|
||||
|
||||
|
||||
def test_backend_warnings_quiet_when_bash_and_shellcheck_both_present(tmp_path, monkeypatch):
|
||||
"""Both installed → no warning."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
from agent.lsp import cli as lsp_cli
|
||||
|
||||
def which(name):
|
||||
return f"/usr/bin/{name}" # both found
|
||||
|
||||
with patch("shutil.which", side_effect=which):
|
||||
notes = lsp_cli._backend_warnings()
|
||||
assert notes == []
|
||||
|
||||
|
||||
def test_backend_warnings_fires_when_bash_installed_but_shellcheck_missing(tmp_path, monkeypatch):
|
||||
"""The exact scenario from the bug report."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
from agent.lsp import cli as lsp_cli
|
||||
|
||||
def which(name):
|
||||
if name == "bash-language-server":
|
||||
return "/fake/bin/bash-language-server"
|
||||
return None # shellcheck missing
|
||||
|
||||
with patch("shutil.which", side_effect=which):
|
||||
notes = lsp_cli._backend_warnings()
|
||||
assert len(notes) == 1
|
||||
assert "shellcheck" in notes[0].lower()
|
||||
assert "bash-language-server" in notes[0].lower()
|
||||
|
||||
|
||||
def test_status_output_includes_backend_warnings_section(tmp_path, monkeypatch):
|
||||
"""End-to-end: status command output includes the warning section."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
# Pretend bash-language-server is installed but shellcheck is missing
|
||||
def which(name):
|
||||
if name == "bash-language-server":
|
||||
return "/fake/bin/bash-language-server"
|
||||
return None
|
||||
|
||||
from agent.lsp import cli as lsp_cli
|
||||
|
||||
buf = io.StringIO()
|
||||
with patch("shutil.which", side_effect=which), redirect_stdout(buf):
|
||||
lsp_cli._cmd_status(emit_json=False)
|
||||
|
||||
output = buf.getvalue()
|
||||
assert "Backend warnings" in output
|
||||
assert "shellcheck" in output
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 3: tier-1 lint treats unusable linters as ``skipped``, not ``error``
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_npx_tsc_missing_treated_as_skipped():
|
||||
"""The original bug: ``npx tsc`` errors when tsc isn't installed.
|
||||
|
||||
Without this fix, the lint result is ``error``, which means the LSP
|
||||
semantic tier (gated on ``success or skipped``) is skipped — the user
|
||||
gets a useless tooling-error message instead of real diagnostics.
|
||||
"""
|
||||
from tools.file_operations import _looks_like_linter_unusable
|
||||
|
||||
npx_failure_output = (
|
||||
" \n"
|
||||
" This is not the tsc command you are looking for \n"
|
||||
" \n"
|
||||
"\n"
|
||||
"To get access to the TypeScript compiler, tsc, from the command line either:\n"
|
||||
"- Use npm install typescript to first add TypeScript to your project before using npx\n"
|
||||
)
|
||||
|
||||
assert _looks_like_linter_unusable("npx", npx_failure_output) is True
|
||||
|
||||
|
||||
def test_real_lint_error_not_classified_as_unusable():
|
||||
"""A genuine TypeScript type error must NOT be misclassified."""
|
||||
from tools.file_operations import _looks_like_linter_unusable
|
||||
|
||||
real_error = (
|
||||
"bad.ts:5:1 - error TS2322: Type 'number' is not assignable to type 'string'.\n"
|
||||
"5 const x: string = greet(42);\n"
|
||||
" ~~~~~~~~~~~~~~~\n"
|
||||
)
|
||||
|
||||
assert _looks_like_linter_unusable("npx", real_error) is False
|
||||
|
||||
|
||||
def test_unknown_base_cmd_returns_false():
|
||||
"""Unfamiliar linters fall through and use the normal error path."""
|
||||
from tools.file_operations import _looks_like_linter_unusable
|
||||
|
||||
assert _looks_like_linter_unusable("eslint", "any output") is False
|
||||
assert _looks_like_linter_unusable("", "anything") is False
|
||||
|
||||
|
||||
def test_check_lint_returns_skipped_when_npx_tsc_unusable(tmp_path):
|
||||
"""Integration: _check_lint sees npx exit non-zero with the npx banner
|
||||
and returns a ``skipped`` LintResult so LSP can still run."""
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
ts_file = tmp_path / "bad.ts"
|
||||
ts_file.write_text("const x: string = 42;\n")
|
||||
|
||||
env = LocalEnvironment()
|
||||
fops = ShellFileOperations(env)
|
||||
|
||||
# Patch _exec to simulate ``npx tsc`` failing because tsc is missing.
|
||||
npx_banner = (
|
||||
" \n"
|
||||
" This is not the tsc command you are looking for \n"
|
||||
)
|
||||
|
||||
def fake_exec(cmd, **kwargs):
|
||||
result = MagicMock()
|
||||
result.exit_code = 1
|
||||
result.stdout = npx_banner
|
||||
return result
|
||||
|
||||
with patch.object(fops, "_exec", side_effect=fake_exec), \
|
||||
patch.object(fops, "_has_command", return_value=True):
|
||||
lint = fops._check_lint(str(ts_file))
|
||||
|
||||
assert lint.skipped is True, (
|
||||
f"expected skipped (so LSP runs); got success={lint.success}, "
|
||||
f"output={lint.output!r}"
|
||||
)
|
||||
assert "not usable" in (lint.message or "")
|
||||
|
||||
|
||||
def test_check_lint_returns_error_for_real_ts_type_errors(tmp_path):
|
||||
"""Sanity: real TypeScript errors still go through the error path."""
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.file_operations import ShellFileOperations
|
||||
|
||||
ts_file = tmp_path / "bad.ts"
|
||||
ts_file.write_text("const x: string = 42;\n")
|
||||
|
||||
env = LocalEnvironment()
|
||||
fops = ShellFileOperations(env)
|
||||
|
||||
real_tsc_error = (
|
||||
"bad.ts:1:7 - error TS2322: Type 'number' is not assignable to type 'string'.\n"
|
||||
"1 const x: string = 42;\n"
|
||||
" ~\n"
|
||||
"Found 1 error.\n"
|
||||
)
|
||||
|
||||
def fake_exec(cmd, **kwargs):
|
||||
result = MagicMock()
|
||||
result.exit_code = 1
|
||||
result.stdout = real_tsc_error
|
||||
return result
|
||||
|
||||
with patch.object(fops, "_exec", side_effect=fake_exec), \
|
||||
patch.object(fops, "_has_command", return_value=True):
|
||||
lint = fops._check_lint(str(ts_file))
|
||||
|
||||
assert lint.skipped is False
|
||||
assert lint.success is False
|
||||
assert "TS2322" in lint.output
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Tests for service-singleton lifecycle: atexit handler, idempotent shutdown.
|
||||
|
||||
These cover the exit-cleanup behavior added to plug the language-server
|
||||
process leak — without the atexit hook, ``hermes chat`` exits while
|
||||
pyright/gopls/etc. are still alive on the host.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import lsp as lsp_module
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_singleton():
|
||||
"""Force a clean module state before each test.
|
||||
|
||||
Tests in this file share process-global state (the lazy
|
||||
singleton + atexit registration flag); reset both before and
|
||||
after every test so order doesn't matter.
|
||||
"""
|
||||
lsp_module._service = None
|
||||
lsp_module._atexit_registered = False
|
||||
yield
|
||||
lsp_module._service = None
|
||||
lsp_module._atexit_registered = False
|
||||
|
||||
|
||||
def test_get_service_registers_atexit_handler_once(monkeypatch):
|
||||
"""First call to ``get_service`` must register an atexit handler;
|
||||
subsequent calls must NOT register another one (Python's ``atexit``
|
||||
runs every registered callable, so a duplicate would shutdown
|
||||
twice — harmless but wasteful)."""
|
||||
fake_svc = MagicMock()
|
||||
fake_svc.is_active.return_value = True
|
||||
monkeypatch.setattr(
|
||||
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
|
||||
)
|
||||
|
||||
registrations = []
|
||||
|
||||
def fake_register(fn):
|
||||
registrations.append(fn)
|
||||
|
||||
monkeypatch.setattr(atexit, "register", fake_register)
|
||||
|
||||
a = lsp_module.get_service()
|
||||
b = lsp_module.get_service()
|
||||
c = lsp_module.get_service()
|
||||
|
||||
assert a is fake_svc
|
||||
assert b is fake_svc
|
||||
assert c is fake_svc
|
||||
assert len(registrations) == 1
|
||||
# The registered callable must be our internal shutdown wrapper.
|
||||
assert registrations[0] is lsp_module._atexit_shutdown
|
||||
|
||||
|
||||
def test_atexit_shutdown_calls_shutdown_service(monkeypatch):
|
||||
"""The atexit-registered wrapper invokes ``shutdown_service`` and
|
||||
swallows any exception — by the time atexit fires, the user has
|
||||
already seen the response and a noisy traceback would be clutter."""
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
lsp_module, "shutdown_service", lambda: called.append("shutdown")
|
||||
)
|
||||
lsp_module._atexit_shutdown()
|
||||
assert called == ["shutdown"]
|
||||
|
||||
|
||||
def test_atexit_shutdown_swallows_exceptions(monkeypatch):
|
||||
def boom():
|
||||
raise RuntimeError("server already dead")
|
||||
|
||||
monkeypatch.setattr(lsp_module, "shutdown_service", boom)
|
||||
# Must not raise.
|
||||
lsp_module._atexit_shutdown()
|
||||
|
||||
|
||||
def test_shutdown_service_idempotent(monkeypatch):
|
||||
"""Calling shutdown twice must be safe — first call cleans up,
|
||||
second call no-ops (nothing to shut down)."""
|
||||
fake_svc = MagicMock()
|
||||
fake_svc.is_active.return_value = True
|
||||
fake_svc.shutdown = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
|
||||
)
|
||||
monkeypatch.setattr(atexit, "register", lambda fn: None)
|
||||
|
||||
lsp_module.get_service()
|
||||
lsp_module.shutdown_service()
|
||||
lsp_module.shutdown_service() # must not raise
|
||||
|
||||
assert fake_svc.shutdown.call_count == 1
|
||||
|
||||
|
||||
def test_shutdown_service_no_op_when_never_started():
|
||||
"""Calling shutdown without ever creating the service is safe."""
|
||||
lsp_module.shutdown_service() # must not raise
|
||||
|
||||
|
||||
def test_shutdown_service_swallows_exception(monkeypatch):
|
||||
"""An exception during ``svc.shutdown()`` must not propagate —
|
||||
the caller (often atexit) has nothing useful to do with it."""
|
||||
fake_svc = MagicMock()
|
||||
fake_svc.is_active.return_value = True
|
||||
fake_svc.shutdown = MagicMock(side_effect=RuntimeError("kill -9 already"))
|
||||
monkeypatch.setattr(
|
||||
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
|
||||
)
|
||||
monkeypatch.setattr(atexit, "register", lambda fn: None)
|
||||
|
||||
lsp_module.get_service()
|
||||
lsp_module.shutdown_service() # must not raise
|
||||
|
||||
|
||||
def test_get_service_returns_none_for_inactive_service(monkeypatch):
|
||||
"""A service whose ``is_active()`` returns False is treated as
|
||||
not running — callers see ``None`` and fall back."""
|
||||
fake_svc = MagicMock()
|
||||
fake_svc.is_active.return_value = False
|
||||
monkeypatch.setattr(
|
||||
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: fake_svc)
|
||||
)
|
||||
monkeypatch.setattr(atexit, "register", lambda fn: None)
|
||||
|
||||
assert lsp_module.get_service() is None
|
||||
# Subsequent call returns None too — but the inactive instance is
|
||||
# cached so we don't re-build it on every check.
|
||||
assert lsp_module.get_service() is None
|
||||
|
||||
|
||||
def test_get_service_returns_none_when_create_fails(monkeypatch):
|
||||
"""Service factory returning ``None`` (no config, etc.) propagates."""
|
||||
monkeypatch.setattr(
|
||||
lsp_module.LSPService, "create_from_config", classmethod(lambda cls: None)
|
||||
)
|
||||
monkeypatch.setattr(atexit, "register", lambda fn: None)
|
||||
|
||||
assert lsp_module.get_service() is None
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Tests for the LSP protocol framing layer.
|
||||
|
||||
The framer is small but load-bearing — Content-Length parsing is the
|
||||
single most common reason for hand-rolled LSP clients to silently
|
||||
deadlock. These tests exercise:
|
||||
|
||||
- exact wire format of outgoing messages (encode_message)
|
||||
- partial-read tolerance + EOF handling (read_message)
|
||||
- envelope helpers (request, response, notification, error)
|
||||
- message classification
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from agent.lsp.protocol import (
|
||||
ERROR_CONTENT_MODIFIED,
|
||||
ERROR_METHOD_NOT_FOUND,
|
||||
LSPProtocolError,
|
||||
LSPRequestError,
|
||||
classify_message,
|
||||
encode_message,
|
||||
make_error_response,
|
||||
make_notification,
|
||||
make_request,
|
||||
make_response,
|
||||
read_message,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# encode_message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_encode_message_uses_compact_separators_and_utf8():
|
||||
msg = {"jsonrpc": "2.0", "id": 1, "method": "x", "params": {"k": "ä"}}
|
||||
out = encode_message(msg)
|
||||
# Header is plain ASCII Content-Length CRLF CRLF
|
||||
header_end = out.index(b"\r\n\r\n") + 4
|
||||
header = out[:header_end].decode("ascii")
|
||||
body = out[header_end:]
|
||||
assert "Content-Length:" in header
|
||||
declared = int(header.split("Content-Length:")[1].split("\r\n")[0].strip())
|
||||
# Declared length must equal actual body bytes.
|
||||
assert declared == len(body)
|
||||
# Body parses as JSON and round-trips.
|
||||
parsed = json.loads(body.decode("utf-8"))
|
||||
assert parsed == msg
|
||||
# Body uses compact separators (no spaces between kv).
|
||||
assert b'"id":1' in body
|
||||
|
||||
|
||||
def test_encode_message_handles_unicode_in_strings():
|
||||
msg = {"jsonrpc": "2.0", "method": "log", "params": {"text": "🚀 ünıcödé"}}
|
||||
out = encode_message(msg)
|
||||
header_end = out.index(b"\r\n\r\n") + 4
|
||||
declared = int(out[: out.index(b"\r\n")].split(b": ")[1])
|
||||
assert declared == len(out[header_end:])
|
||||
assert json.loads(out[header_end:].decode("utf-8")) == msg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# read_message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _stream_from_bytes(data: bytes) -> asyncio.StreamReader:
|
||||
"""Build an asyncio.StreamReader pre-populated with ``data``."""
|
||||
reader = asyncio.StreamReader()
|
||||
reader.feed_data(data)
|
||||
reader.feed_eof()
|
||||
return reader
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_message_round_trip():
|
||||
msg = {"jsonrpc": "2.0", "method": "ping"}
|
||||
reader = await _stream_from_bytes(encode_message(msg))
|
||||
parsed = await read_message(reader)
|
||||
assert parsed == msg
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_message_clean_eof_returns_none():
|
||||
reader = await _stream_from_bytes(b"")
|
||||
assert await read_message(reader) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_message_truncated_body_raises():
|
||||
msg = encode_message({"jsonrpc": "2.0", "method": "x"})
|
||||
truncated = msg[: -3] # cut the body
|
||||
reader = await _stream_from_bytes(truncated)
|
||||
with pytest.raises(LSPProtocolError):
|
||||
await read_message(reader)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_message_missing_content_length_raises():
|
||||
bad = b"X-Other: 5\r\n\r\n12345"
|
||||
reader = await _stream_from_bytes(bad)
|
||||
with pytest.raises(LSPProtocolError):
|
||||
await read_message(reader)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_message_two_messages_back_to_back():
|
||||
a = encode_message({"jsonrpc": "2.0", "method": "a"})
|
||||
b = encode_message({"jsonrpc": "2.0", "method": "b"})
|
||||
reader = await _stream_from_bytes(a + b)
|
||||
assert (await read_message(reader))["method"] == "a"
|
||||
assert (await read_message(reader))["method"] == "b"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_message_rejects_runaway_header():
|
||||
"""A pathological server that streams headers without ever emitting
|
||||
the CRLF-CRLF terminator must not loop forever — the 8 KiB cap kicks
|
||||
in and surfaces a protocol error."""
|
||||
flood = (b"X-Junk: " + b"A" * 200 + b"\r\n") * 60 # ~12 KiB worth
|
||||
reader = await _stream_from_bytes(flood)
|
||||
with pytest.raises(LSPProtocolError) as exc:
|
||||
await read_message(reader)
|
||||
assert "8 KiB" in str(exc.value)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# envelope helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_make_request_includes_id_and_method():
|
||||
msg = make_request(7, "ping", {"v": 1})
|
||||
assert msg == {"jsonrpc": "2.0", "id": 7, "method": "ping", "params": {"v": 1}}
|
||||
|
||||
|
||||
def test_make_request_omits_params_when_none():
|
||||
msg = make_request(7, "ping", None)
|
||||
assert "params" not in msg
|
||||
|
||||
|
||||
def test_make_notification_omits_id():
|
||||
msg = make_notification("log", {"line": "hi"})
|
||||
assert "id" not in msg
|
||||
assert msg["method"] == "log"
|
||||
|
||||
|
||||
def test_make_response_carries_result():
|
||||
msg = make_response(7, {"ok": True})
|
||||
assert msg["id"] == 7 and msg["result"] == {"ok": True}
|
||||
|
||||
|
||||
def test_make_error_response_shape():
|
||||
msg = make_error_response(7, ERROR_CONTENT_MODIFIED, "stale", {"hint": "retry"})
|
||||
assert msg["error"]["code"] == ERROR_CONTENT_MODIFIED
|
||||
assert msg["error"]["message"] == "stale"
|
||||
assert msg["error"]["data"] == {"hint": "retry"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# classify_message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_classify_message_request():
|
||||
msg = {"jsonrpc": "2.0", "id": 1, "method": "x"}
|
||||
assert classify_message(msg) == ("request", 1)
|
||||
|
||||
|
||||
def test_classify_message_response():
|
||||
msg = {"jsonrpc": "2.0", "id": 1, "result": None}
|
||||
assert classify_message(msg) == ("response", 1)
|
||||
|
||||
|
||||
def test_classify_message_notification():
|
||||
msg = {"jsonrpc": "2.0", "method": "log"}
|
||||
assert classify_message(msg) == ("notification", "log")
|
||||
|
||||
|
||||
def test_classify_message_invalid():
|
||||
assert classify_message({"id": 1})[0] == "invalid"
|
||||
assert classify_message({"jsonrpc": "1.0", "method": "x"})[0] == "invalid"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LSPRequestError
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_lsp_request_error_carries_code_and_data():
|
||||
e = LSPRequestError(ERROR_METHOD_NOT_FOUND, "no", {"x": 1})
|
||||
assert e.code == ERROR_METHOD_NOT_FOUND
|
||||
assert e.message == "no"
|
||||
assert e.data == {"x": 1}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Tests for the diagnostic reporter (formatting layer)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from agent.lsp.reporter import (
|
||||
MAX_PER_FILE,
|
||||
format_diagnostic,
|
||||
report_for_file,
|
||||
truncate,
|
||||
)
|
||||
|
||||
|
||||
def _diag(line=0, col=0, sev=1, code="E001", source="ls", msg="oops"):
|
||||
return {
|
||||
"range": {
|
||||
"start": {"line": line, "character": col},
|
||||
"end": {"line": line, "character": col + 1},
|
||||
},
|
||||
"severity": sev,
|
||||
"code": code,
|
||||
"source": source,
|
||||
"message": msg,
|
||||
}
|
||||
|
||||
|
||||
def test_format_diagnostic_uses_one_indexed_position():
|
||||
line = format_diagnostic(_diag(line=4, col=2))
|
||||
assert "[5:3]" in line # +1 on both
|
||||
|
||||
|
||||
def test_format_diagnostic_includes_severity_label():
|
||||
assert format_diagnostic(_diag(sev=1)).startswith("ERROR")
|
||||
assert format_diagnostic(_diag(sev=2)).startswith("WARN")
|
||||
assert format_diagnostic(_diag(sev=3)).startswith("INFO")
|
||||
assert format_diagnostic(_diag(sev=4)).startswith("HINT")
|
||||
|
||||
|
||||
def test_format_diagnostic_includes_code_and_source():
|
||||
line = format_diagnostic(_diag(code="X42", source="src"))
|
||||
assert "[X42]" in line
|
||||
assert "(src)" in line
|
||||
|
||||
|
||||
def test_format_diagnostic_omits_missing_optional_fields():
|
||||
line = format_diagnostic(
|
||||
{
|
||||
"range": {
|
||||
"start": {"line": 0, "character": 0},
|
||||
"end": {"line": 0, "character": 0},
|
||||
},
|
||||
"severity": 1,
|
||||
"message": "bare",
|
||||
}
|
||||
)
|
||||
assert "[" not in line.split("]", 1)[1] # no extra brackets after the position
|
||||
assert "(" not in line
|
||||
|
||||
|
||||
def test_report_for_file_returns_empty_when_only_warnings():
|
||||
"""Default severity filter is ERROR-only."""
|
||||
report = report_for_file("/x.py", [_diag(sev=2)])
|
||||
assert report == ""
|
||||
|
||||
|
||||
def test_report_for_file_emits_block_with_errors():
|
||||
diag = _diag(msg="real error")
|
||||
report = report_for_file("/x.py", [diag])
|
||||
assert "<diagnostics file=\"/x.py\">" in report
|
||||
assert "real error" in report
|
||||
assert "</diagnostics>" in report
|
||||
|
||||
|
||||
def test_report_for_file_caps_at_max_per_file():
|
||||
diags = [_diag(line=i) for i in range(MAX_PER_FILE + 5)]
|
||||
report = report_for_file("/x.py", diags)
|
||||
assert "and 5 more" in report
|
||||
|
||||
|
||||
def test_report_for_file_respects_custom_severities():
|
||||
diag = _diag(sev=2, msg="warn")
|
||||
report = report_for_file("/x.py", [diag], severities=frozenset({1, 2}))
|
||||
assert "warn" in report
|
||||
|
||||
|
||||
def test_truncate_below_limit_unchanged():
|
||||
s = "abc" * 100
|
||||
assert truncate(s, limit=4000) == s
|
||||
|
||||
|
||||
def test_truncate_above_limit_appends_marker():
|
||||
s = "x" * 10000
|
||||
out = truncate(s, limit=200)
|
||||
assert out.endswith("[truncated]")
|
||||
assert len(out) <= 200
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Tests for the synchronous LSPService wrapper.
|
||||
|
||||
Drives the service through ``snapshot_baseline`` →
|
||||
``get_diagnostics_sync`` against the mock LSP server, exercising the
|
||||
delta filter that ``tools/file_operations._check_lint_delta`` relies
|
||||
on.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp.manager import LSPService
|
||||
from agent.lsp.servers import (
|
||||
SERVERS,
|
||||
ServerContext,
|
||||
ServerDef,
|
||||
SpawnSpec,
|
||||
)
|
||||
|
||||
|
||||
MOCK_SERVER = str(Path(__file__).parent / "_mock_lsp_server.py")
|
||||
|
||||
|
||||
def _install_mock_server(monkeypatch, script: str = "errors", server_id: str = "pyright"):
|
||||
"""Replace one registered server with a wrapper that spawns the mock.
|
||||
|
||||
We reuse ``pyright`` so .py files route to it. This keeps the
|
||||
test free of any LSP toolchain dependency.
|
||||
"""
|
||||
target_index = next(i for i, s in enumerate(SERVERS) if s.server_id == server_id)
|
||||
original = SERVERS[target_index]
|
||||
|
||||
def _spawn(root: str, ctx: ServerContext) -> SpawnSpec:
|
||||
env = {"MOCK_LSP_SCRIPT": script}
|
||||
return SpawnSpec(
|
||||
command=[sys.executable, MOCK_SERVER],
|
||||
workspace_root=root,
|
||||
cwd=root,
|
||||
env=env,
|
||||
initialization_options={},
|
||||
)
|
||||
|
||||
replacement = ServerDef(
|
||||
server_id=server_id,
|
||||
extensions=original.extensions,
|
||||
resolve_root=lambda fp, ws: ws, # always use workspace root
|
||||
build_spawn=_spawn,
|
||||
seed_first_push=False,
|
||||
description="mock " + server_id,
|
||||
)
|
||||
# Patch the SERVERS list element directly + restore on teardown.
|
||||
SERVERS[target_index] = replacement
|
||||
|
||||
yield
|
||||
|
||||
SERVERS[target_index] = original
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_pyright(monkeypatch, tmp_path):
|
||||
"""Install the mock as ``pyright`` and create a fake git workspace."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / ".git").mkdir()
|
||||
(repo / "pyproject.toml").write_text("") # so pyright's root resolver finds it
|
||||
monkeypatch.chdir(str(repo))
|
||||
gen = _install_mock_server(monkeypatch, "errors", "pyright")
|
||||
next(gen)
|
||||
yield repo
|
||||
try:
|
||||
next(gen)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
|
||||
def test_service_returns_empty_when_disabled(tmp_path):
|
||||
svc = LSPService(
|
||||
enabled=False,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="auto",
|
||||
)
|
||||
assert not svc.is_active()
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
assert svc.get_diagnostics_sync(str(f)) == []
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_skips_files_outside_workspace(tmp_path):
|
||||
"""Files outside any git worktree must not trigger LSP."""
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=2.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
# No .git anywhere — service should report not enabled for this file.
|
||||
assert not svc.enabled_for(str(f))
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_e2e_delta_filter(mock_pyright):
|
||||
"""End-to-end: snapshot baseline → wait → delta returned."""
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=3.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
assert svc.enabled_for(str(f))
|
||||
# Baseline first — server pushes 1 error.
|
||||
svc.snapshot_baseline(str(f))
|
||||
# Re-poll: same error is in baseline, so delta is empty.
|
||||
new_diags = svc.get_diagnostics_sync(str(f))
|
||||
assert new_diags == []
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_e2e_delta_filter_with_line_shift(mock_pyright):
|
||||
"""End-to-end: an edit that shifts the diagnostic's line still
|
||||
filters correctly when ``line_shift`` is supplied.
|
||||
|
||||
The mock LSP server emits a fixed error at line 0; for this test
|
||||
we don't need to actually shift the server's output — we just
|
||||
need to prove that supplying a line_shift through the API works
|
||||
and doesn't break the existing delta path. The unit tests in
|
||||
test_delta_key.py cover the shift semantics in detail.
|
||||
"""
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=3.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc.snapshot_baseline(str(f))
|
||||
# Identity shift — should behave exactly like no shift.
|
||||
new_diags = svc.get_diagnostics_sync(str(f), line_shift=lambda L: L)
|
||||
assert new_diags == []
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_status_includes_clients(mock_pyright):
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
f.write_text("")
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=3.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc.get_diagnostics_sync(str(f))
|
||||
info = svc.get_status()
|
||||
assert info["enabled"] is True
|
||||
assert any(c["server_id"] == "pyright" for c in info["clients"])
|
||||
finally:
|
||||
svc.shutdown()
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Skip the per-file shell linter when LSP will handle the same file.
|
||||
|
||||
The per-file ``npx tsc --noEmit FILE.ts`` shell linter cannot see
|
||||
``tsconfig.json`` (a documented ``tsc`` quirk: explicit file args bypass
|
||||
the project config), so it defaults to no-lib / ES5 and floods the
|
||||
agent's lint field with phantom "Cannot find 'Promise' / 'Map' / 'Set' /
|
||||
'ReadonlySet' / 'Iterable' / 'imul' / …" errors on every edit — up to
|
||||
25K tokens per patch. The LSP tier (``tsserver`` via
|
||||
typescript-language-server) reads tsconfig correctly and surfaces real
|
||||
diagnostics in the ``lsp_diagnostics`` field of the WriteResult /
|
||||
PatchResult.
|
||||
|
||||
These tests pin the contract:
|
||||
|
||||
- When LSP is active AND ``enabled_for(path)`` for a ``.ts`` / ``.go``
|
||||
/ ``.rs`` file, ``_check_lint`` returns ``skipped`` without invoking
|
||||
the shell linter at all.
|
||||
- When LSP is inactive or disabled-for-path, the shell linter runs
|
||||
exactly as before (regression guard for the default config).
|
||||
- The skip only applies to extensions in
|
||||
``_SHELL_LINTER_LSP_REDUNDANT`` — Python ``py_compile`` and
|
||||
``node --check`` keep running unconditionally because they're fast,
|
||||
file-local, and correct.
|
||||
- ``.tsx`` is intentionally NOT in either ``LINTERS`` or
|
||||
``_SHELL_LINTER_LSP_REDUNDANT``: it had no ``LINTERS`` entry
|
||||
pre-PR (so it was already implicitly ``skipped`` via the
|
||||
``ext not in LINTERS`` branch) and adding one would have inherited
|
||||
``.ts``'s broken ``tsc --noEmit FILE`` invocation for LSP-disabled
|
||||
users. When LSP IS enabled, ``.tsx`` is still covered by
|
||||
typescript-language-server via ``_maybe_lsp_diagnostics`` — the
|
||||
diagnostics show up on ``lsp_diagnostics``, not ``lint``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _make_fops():
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.file_operations import ShellFileOperations
|
||||
return ShellFileOperations(LocalEnvironment())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ext", [".ts", ".go", ".rs"])
|
||||
def test_shell_linter_skipped_when_lsp_will_handle(ext, tmp_path):
|
||||
"""When LSP is active and enabled_for(path), shell linter is skipped.
|
||||
|
||||
The shell linter's _exec must NOT be called — that's the whole
|
||||
point. We assert by patching ``_exec`` to raise, so any accidental
|
||||
invocation surfaces as a test failure.
|
||||
"""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / f"bad{ext}"
|
||||
src.write_text("intentionally invalid content\n")
|
||||
|
||||
def _exec_must_not_run(*args, **kwargs): # pragma: no cover
|
||||
raise AssertionError(
|
||||
"shell linter was invoked despite LSP claiming the file"
|
||||
)
|
||||
|
||||
with patch.object(fops, "_lsp_will_handle", return_value=True), \
|
||||
patch.object(fops, "_exec", side_effect=_exec_must_not_run), \
|
||||
patch.object(fops, "_has_command", return_value=True):
|
||||
result = fops._check_lint(str(src))
|
||||
|
||||
assert result.skipped is True
|
||||
assert "LSP" in (result.message or "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ext", [".ts", ".go", ".rs"])
|
||||
def test_shell_linter_runs_when_lsp_inactive(ext, tmp_path):
|
||||
"""When LSP is inactive (default config, no service, remote backend, ...),
|
||||
the shell linter runs as before — no behavior change."""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / f"clean{ext}"
|
||||
src.write_text("// content\n")
|
||||
|
||||
fake_result = MagicMock()
|
||||
fake_result.exit_code = 0
|
||||
fake_result.stdout = ""
|
||||
|
||||
with patch.object(fops, "_lsp_will_handle", return_value=False), \
|
||||
patch.object(fops, "_exec", return_value=fake_result) as exec_mock, \
|
||||
patch.object(fops, "_has_command", return_value=True):
|
||||
result = fops._check_lint(str(src))
|
||||
|
||||
# _exec must have been called — proving the shell linter ran.
|
||||
assert exec_mock.called, "shell linter did NOT run when LSP was inactive"
|
||||
assert result.success is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ext", [".py", ".js"])
|
||||
def test_lsp_does_not_skip_non_redundant_extensions(ext, tmp_path):
|
||||
"""``py_compile`` and ``node --check`` keep running even when an LSP
|
||||
server (pyright/pylsp/typescript-language-server-for-JS) is active —
|
||||
they're fast, file-local, and correct, so there's no upside to
|
||||
suppressing them.
|
||||
"""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / f"clean{ext}"
|
||||
src.write_text("# valid\n" if ext == ".py" else "// valid\n")
|
||||
|
||||
fake_result = MagicMock()
|
||||
fake_result.exit_code = 0
|
||||
fake_result.stdout = ""
|
||||
|
||||
# Even with LSP claiming the file, the shell linter must still run
|
||||
# for these extensions.
|
||||
with patch.object(fops, "_lsp_will_handle", return_value=True), \
|
||||
patch.object(fops, "_exec", return_value=fake_result) as exec_mock, \
|
||||
patch.object(fops, "_has_command", return_value=True):
|
||||
fops._check_lint(str(src))
|
||||
|
||||
assert exec_mock.called, (
|
||||
f"shell linter for {ext} did not run despite being in the "
|
||||
"'always-run' set (py_compile / node --check)"
|
||||
)
|
||||
|
||||
|
||||
def test_lsp_will_handle_returns_false_when_service_is_none(tmp_path):
|
||||
"""``_lsp_will_handle`` must return False when the LSP service hasn't
|
||||
been initialized — otherwise we'd accidentally skip the shell linter
|
||||
on systems where LSP isn't configured at all."""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / "foo.ts"
|
||||
src.write_text("const x = 1\n")
|
||||
|
||||
with patch.object(fops, "_lsp_local_only", return_value=True), \
|
||||
patch("agent.lsp.get_service", return_value=None):
|
||||
assert fops._lsp_will_handle(str(src)) is False
|
||||
|
||||
|
||||
def test_lsp_will_handle_returns_false_on_remote_backend(tmp_path):
|
||||
"""LSP servers run on the host process — remote backends (Docker,
|
||||
SSH, Modal, …) keep files inside the sandbox where the host LSP
|
||||
can't reach them. ``_lsp_will_handle`` must short-circuit before
|
||||
calling into the service in that case."""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / "foo.ts"
|
||||
src.write_text("const x = 1\n")
|
||||
|
||||
with patch.object(fops, "_lsp_local_only", return_value=False), \
|
||||
patch("agent.lsp.get_service") as get_service_mock:
|
||||
result = fops._lsp_will_handle(str(src))
|
||||
|
||||
assert result is False
|
||||
# Importantly: we never even consulted the service.
|
||||
assert not get_service_mock.called
|
||||
|
||||
|
||||
def test_lsp_will_handle_swallows_enabled_for_exception(tmp_path):
|
||||
"""A flaky LSP service must never break the shell-linter fallback —
|
||||
if ``enabled_for`` raises, we treat the file as "not handled" so the
|
||||
shell linter still runs."""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / "foo.ts"
|
||||
src.write_text("const x = 1\n")
|
||||
|
||||
fake_svc = MagicMock()
|
||||
fake_svc.enabled_for.side_effect = RuntimeError("server crashed")
|
||||
|
||||
with patch.object(fops, "_lsp_local_only", return_value=True), \
|
||||
patch("agent.lsp.get_service", return_value=fake_svc):
|
||||
assert fops._lsp_will_handle(str(src)) is False
|
||||
|
||||
|
||||
def test_tsx_stays_out_of_linters_table_for_default_compatibility():
|
||||
"""Regression: keep ``.tsx`` out of ``LINTERS`` so users with LSP
|
||||
DISABLED don't suddenly get the broken ``npx tsc --noEmit FILE.tsx``
|
||||
invocation that ``.ts`` historically used to get.
|
||||
|
||||
Pre-PR behavior: ``.tsx`` had no entry in ``LINTERS``, so it fell
|
||||
through to ``ext not in LINTERS`` → ``LintResult(skipped=True,
|
||||
message="No linter for .tsx files")``. This PR preserves that for
|
||||
the default config.
|
||||
|
||||
When LSP IS enabled, ``.tsx`` is still covered by the LSP tier via
|
||||
``_maybe_lsp_diagnostics`` (typescript-language-server claims
|
||||
``.tsx`` in its extensions list) — the diagnostics show up in the
|
||||
``lsp_diagnostics`` field, not the ``lint`` field.
|
||||
"""
|
||||
from tools.file_operations import LINTERS, _SHELL_LINTER_LSP_REDUNDANT
|
||||
|
||||
assert ".tsx" not in LINTERS
|
||||
assert ".tsx" not in _SHELL_LINTER_LSP_REDUNDANT
|
||||
|
||||
|
||||
def test_tsx_default_check_lint_returns_skipped(tmp_path):
|
||||
"""End-to-end: ``.tsx`` files get ``LintResult(skipped=True)`` from
|
||||
``_check_lint`` regardless of LSP status — this is the no-regression
|
||||
contract that addresses Copilot review #3271017282."""
|
||||
fops = _make_fops()
|
||||
src = tmp_path / "foo.tsx"
|
||||
src.write_text("export const X = () => <div/>\n")
|
||||
|
||||
# Even with LSP claiming the file, no shell linter runs for .tsx
|
||||
# because there's no LINTERS entry — the ``ext not in LINTERS``
|
||||
# branch fires before the LSP short-circuit is consulted.
|
||||
with patch.object(fops, "_lsp_will_handle", return_value=True), \
|
||||
patch.object(fops, "_exec") as exec_mock:
|
||||
result = fops._check_lint(str(src))
|
||||
|
||||
assert result.skipped is True
|
||||
assert not exec_mock.called, "no shell linter should run for .tsx"
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Tests for workspace + project-root resolution."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.lsp.workspace import (
|
||||
clear_cache,
|
||||
find_git_worktree,
|
||||
is_inside_workspace,
|
||||
nearest_root,
|
||||
normalize_path,
|
||||
resolve_workspace_for_file,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear():
|
||||
clear_cache()
|
||||
yield
|
||||
clear_cache()
|
||||
|
||||
|
||||
def test_find_git_worktree_returns_none_outside_repo(tmp_path: Path):
|
||||
sub = tmp_path / "sub"
|
||||
sub.mkdir()
|
||||
assert find_git_worktree(str(sub)) is None
|
||||
|
||||
|
||||
def test_find_git_worktree_finds_dotgit(tmp_path: Path):
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / ".git").mkdir()
|
||||
sub = repo / "src" / "deep"
|
||||
sub.mkdir(parents=True)
|
||||
assert find_git_worktree(str(sub)) == str(repo)
|
||||
|
||||
|
||||
def test_find_git_worktree_handles_dotgit_file(tmp_path: Path):
|
||||
"""``.git`` can also be a file (gitfile pointing into a worktree)."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / ".git").write_text("gitdir: /elsewhere\n")
|
||||
assert find_git_worktree(str(repo)) == str(repo)
|
||||
|
||||
|
||||
def test_is_inside_workspace_true_for_subpath(tmp_path: Path):
|
||||
root = tmp_path / "p"
|
||||
root.mkdir()
|
||||
sub = root / "x" / "y.py"
|
||||
sub.parent.mkdir(parents=True)
|
||||
sub.write_text("")
|
||||
assert is_inside_workspace(str(sub), str(root))
|
||||
|
||||
|
||||
def test_is_inside_workspace_false_for_unrelated(tmp_path: Path):
|
||||
a = tmp_path / "a"
|
||||
b = tmp_path / "b"
|
||||
a.mkdir()
|
||||
b.mkdir()
|
||||
f = b / "x.py"
|
||||
f.write_text("")
|
||||
assert not is_inside_workspace(str(f), str(a))
|
||||
|
||||
|
||||
def test_nearest_root_finds_first_marker(tmp_path: Path):
|
||||
root = tmp_path / "p"
|
||||
deep = root / "src" / "pkg"
|
||||
deep.mkdir(parents=True)
|
||||
(root / "pyproject.toml").write_text("")
|
||||
found = nearest_root(str(deep / "mod.py"), ["pyproject.toml"])
|
||||
assert found == str(root)
|
||||
|
||||
|
||||
def test_nearest_root_excludes_take_priority(tmp_path: Path):
|
||||
"""If an exclude marker matches first, return None."""
|
||||
root = tmp_path / "p"
|
||||
sub = root / "deno-app"
|
||||
sub.mkdir(parents=True)
|
||||
(sub / "deno.json").write_text("{}")
|
||||
(root / "package.json").write_text("{}") # would match if not for exclude
|
||||
found = nearest_root(
|
||||
str(sub / "main.ts"),
|
||||
["package.json"],
|
||||
excludes=["deno.json"],
|
||||
)
|
||||
assert found is None
|
||||
|
||||
|
||||
def test_nearest_root_returns_none_when_no_marker(tmp_path: Path):
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
assert nearest_root(str(f), ["pyproject.toml"]) is None
|
||||
|
||||
|
||||
def test_resolve_workspace_for_file_uses_cwd_first(tmp_path: Path, monkeypatch):
|
||||
repo = tmp_path / "repo"
|
||||
(repo / ".git").mkdir(parents=True)
|
||||
file_path = repo / "x.py"
|
||||
file_path.write_text("")
|
||||
# cwd is inside the repo
|
||||
monkeypatch.chdir(str(repo))
|
||||
root, gated = resolve_workspace_for_file(str(file_path))
|
||||
assert root == str(repo)
|
||||
assert gated is True
|
||||
|
||||
|
||||
def test_resolve_workspace_for_file_no_repo_returns_none(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.chdir(str(tmp_path))
|
||||
f = tmp_path / "x.py"
|
||||
f.write_text("")
|
||||
root, gated = resolve_workspace_for_file(str(f))
|
||||
assert root is None
|
||||
assert gated is False
|
||||
|
||||
|
||||
def test_resolve_workspace_falls_back_to_file_location(tmp_path: Path, monkeypatch):
|
||||
"""When cwd isn't a git repo but the file is inside one, we still
|
||||
discover the workspace from the file's path."""
|
||||
not_a_repo = tmp_path / "loose"
|
||||
not_a_repo.mkdir()
|
||||
monkeypatch.chdir(str(not_a_repo))
|
||||
|
||||
repo = tmp_path / "actual-repo"
|
||||
(repo / ".git").mkdir(parents=True)
|
||||
f = repo / "x.py"
|
||||
f.write_text("")
|
||||
|
||||
root, gated = resolve_workspace_for_file(str(f))
|
||||
assert root == str(repo)
|
||||
assert gated is True
|
||||
|
||||
|
||||
def test_normalize_path_expands_tilde(monkeypatch):
|
||||
monkeypatch.setenv("HOME", "/home/user")
|
||||
p = normalize_path("~/x.py")
|
||||
assert p == os.path.abspath("/home/user/x.py")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
"""Tests for Bug #12905 fixes in agent/anthropic_adapter.py — macOS Keychain support."""
|
||||
|
||||
import json
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
from agent.anthropic_adapter import (
|
||||
_read_claude_code_credentials_from_keychain,
|
||||
read_claude_code_credentials,
|
||||
)
|
||||
|
||||
|
||||
class TestReadClaudeCodeCredentialsFromKeychain:
|
||||
"""Bug 4: macOS Keychain support for Claude Code >=2.1.114."""
|
||||
|
||||
def test_returns_none_on_linux(self):
|
||||
"""Keychain reading is Darwin-only; must return None on other platforms."""
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Linux"):
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_on_windows(self):
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Windows"):
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_when_security_command_not_found(self):
|
||||
"""OSError from missing security binary must be handled gracefully."""
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run",
|
||||
side_effect=OSError("security not found")):
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_on_nonzero_exit_code(self):
|
||||
"""security returns non-zero when the Keychain entry doesn't exist."""
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_for_empty_stdout(self):
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_for_non_json_payload(self):
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="not valid json", stderr="")
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_when_password_field_is_missing_claude_ai_oauth(self):
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"someOtherService": {"accessToken": "tok"}}),
|
||||
stderr="",
|
||||
)
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_returns_none_when_access_token_is_empty(self):
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"claudeAiOauth": {"accessToken": "", "refreshToken": "x"}}),
|
||||
stderr="",
|
||||
)
|
||||
assert _read_claude_code_credentials_from_keychain() is None
|
||||
|
||||
def test_parses_valid_keychain_entry(self):
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0,
|
||||
stdout=json.dumps({
|
||||
"claudeAiOauth": {
|
||||
"accessToken": "kc-access-token-abc",
|
||||
"refreshToken": "kc-refresh-token-xyz",
|
||||
"expiresAt": 9999999999999,
|
||||
}
|
||||
}),
|
||||
stderr="",
|
||||
)
|
||||
creds = _read_claude_code_credentials_from_keychain()
|
||||
assert creds is not None
|
||||
assert creds["accessToken"] == "kc-access-token-abc"
|
||||
assert creds["refreshToken"] == "kc-refresh-token-xyz"
|
||||
assert creds["expiresAt"] == 9999999999999
|
||||
assert creds["source"] == "macos_keychain"
|
||||
|
||||
|
||||
class TestReadClaudeCodeCredentialsPriority:
|
||||
"""Bug 4: Keychain must be checked before the JSON file."""
|
||||
|
||||
def test_keychain_takes_priority_over_json_file(self, tmp_path, monkeypatch):
|
||||
"""When both Keychain and JSON file have credentials, Keychain wins."""
|
||||
# Set up JSON file with "older" token
|
||||
json_cred_file = tmp_path / ".claude" / ".credentials.json"
|
||||
json_cred_file.parent.mkdir(parents=True)
|
||||
json_cred_file.write_text(json.dumps({
|
||||
"claudeAiOauth": {
|
||||
"accessToken": "json-token",
|
||||
"refreshToken": "json-refresh",
|
||||
"expiresAt": 9999999999999,
|
||||
}
|
||||
}))
|
||||
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
|
||||
|
||||
# Mock Keychain to return a "newer" token
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0,
|
||||
stdout=json.dumps({
|
||||
"claudeAiOauth": {
|
||||
"accessToken": "keychain-token",
|
||||
"refreshToken": "keychain-refresh",
|
||||
"expiresAt": 9999999999999,
|
||||
}
|
||||
}),
|
||||
stderr="",
|
||||
)
|
||||
creds = read_claude_code_credentials()
|
||||
|
||||
# Keychain token should be returned, not JSON file token
|
||||
assert creds is not None
|
||||
assert creds["accessToken"] == "keychain-token"
|
||||
assert creds["source"] == "macos_keychain"
|
||||
|
||||
def test_falls_back_to_json_when_keychain_returns_none(self, tmp_path, monkeypatch):
|
||||
"""When Keychain has no entry, JSON file is used as fallback."""
|
||||
json_cred_file = tmp_path / ".claude" / ".credentials.json"
|
||||
json_cred_file.parent.mkdir(parents=True)
|
||||
json_cred_file.write_text(json.dumps({
|
||||
"claudeAiOauth": {
|
||||
"accessToken": "json-fallback-token",
|
||||
"refreshToken": "json-refresh",
|
||||
"expiresAt": 9999999999999,
|
||||
}
|
||||
}))
|
||||
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
|
||||
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
# Simulate Keychain entry not found
|
||||
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
|
||||
creds = read_claude_code_credentials()
|
||||
|
||||
assert creds is not None
|
||||
assert creds["accessToken"] == "json-fallback-token"
|
||||
assert creds["source"] == "claude_code_credentials_file"
|
||||
|
||||
def test_returns_none_when_neither_keychain_nor_json_has_creds(self, tmp_path, monkeypatch):
|
||||
"""No credentials anywhere — must return None cleanly."""
|
||||
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
|
||||
|
||||
with patch("agent.anthropic_adapter.platform.system", return_value="Darwin"), \
|
||||
patch("agent.anthropic_adapter.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
|
||||
creds = read_claude_code_credentials()
|
||||
|
||||
assert creds is None
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Tests for GH-25255: Anthropic OAuth mcp_ prefix stripping.
|
||||
|
||||
When strip_tool_prefix=True (Anthropic OAuth path), the transport must only
|
||||
strip the ``mcp_`` prefix from OAuth-injected tools, NOT from Hermes-native
|
||||
MCP server tools that are registered under their full ``mcp_<server>_<tool>``
|
||||
name in the tool registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_tool_use_block(name: str, block_id: str = "tc_1", input_data: dict | None = None):
|
||||
"""Create a fake Anthropic tool_use content block."""
|
||||
return SimpleNamespace(
|
||||
type="tool_use",
|
||||
id=block_id,
|
||||
name=name,
|
||||
input=input_data or {"query": "test"},
|
||||
)
|
||||
|
||||
|
||||
def _make_response(*blocks, stop_reason="end_turn"):
|
||||
"""Create a fake Anthropic Messages response."""
|
||||
return SimpleNamespace(
|
||||
content=list(blocks),
|
||||
stop_reason=stop_reason,
|
||||
model="claude-sonnet-4",
|
||||
usage=SimpleNamespace(input_tokens=100, output_tokens=50),
|
||||
)
|
||||
|
||||
|
||||
class _FakeRegistry:
|
||||
"""Minimal fake tool registry for testing prefix stripping logic."""
|
||||
|
||||
def __init__(self, registered_names: set[str]):
|
||||
self._names = registered_names
|
||||
|
||||
def get_entry(self, name: str):
|
||||
if name in self._names:
|
||||
return SimpleNamespace(name=name) # truthy = tool exists
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestAnthropicMcpPrefixStrip:
|
||||
"""Verify that strip_tool_prefix only strips OAuth-injected prefixes."""
|
||||
|
||||
def _get_transport(self):
|
||||
from agent.transports.anthropic import AnthropicTransport
|
||||
return AnthropicTransport()
|
||||
|
||||
def test_strips_prefix_for_oauth_injected_tool(self):
|
||||
"""OAuth tools: mcp_read_file -> read_file (stripped).
|
||||
|
||||
The tool was registered as 'read_file' in the registry.
|
||||
Anthropic sees 'mcp_read_file' because Hermes adds the prefix.
|
||||
On response, we must strip it back to 'read_file'.
|
||||
"""
|
||||
transport = self._get_transport()
|
||||
block = _make_tool_use_block("mcp_read_file")
|
||||
response = _make_response(block)
|
||||
|
||||
registry = _FakeRegistry({"read_file", "terminal", "web_search"})
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=True)
|
||||
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "read_file"
|
||||
|
||||
def test_preserves_native_mcp_server_tool_name(self):
|
||||
"""Native MCP tools: mcp_composio_SEARCH -> mcp_composio_SEARCH (kept).
|
||||
|
||||
The tool is registered with the full mcp_ prefix in the registry.
|
||||
Stripping would break registry lookup.
|
||||
"""
|
||||
transport = self._get_transport()
|
||||
block = _make_tool_use_block("mcp_composio_COMPOSIO_SEARCH_TOOLS")
|
||||
response = _make_response(block)
|
||||
|
||||
registry = _FakeRegistry({
|
||||
"mcp_composio_COMPOSIO_SEARCH_TOOLS",
|
||||
"mcp_composio_COMPOSIO_GET_TOOL_SCHEMAS",
|
||||
"read_file",
|
||||
})
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=True)
|
||||
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "mcp_composio_COMPOSIO_SEARCH_TOOLS"
|
||||
|
||||
def test_no_strip_when_flag_false(self):
|
||||
"""When strip_tool_prefix=False, names are never modified."""
|
||||
transport = self._get_transport()
|
||||
block = _make_tool_use_block("mcp_read_file")
|
||||
response = _make_response(block)
|
||||
|
||||
registry = _FakeRegistry({"read_file"})
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=False)
|
||||
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "mcp_read_file"
|
||||
|
||||
def test_no_strip_when_not_mcp_prefixed(self):
|
||||
"""Non-mcp_ names are untouched regardless of strip flag."""
|
||||
transport = self._get_transport()
|
||||
block = _make_tool_use_block("web_search")
|
||||
response = _make_response(block)
|
||||
|
||||
registry = _FakeRegistry({"web_search"})
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=True)
|
||||
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "web_search"
|
||||
|
||||
def test_preserves_name_when_neither_in_registry(self):
|
||||
"""When neither stripped nor full name is in registry, keep full name.
|
||||
|
||||
Safety fallback: if we can't determine the type, prefer the full name
|
||||
since it's what the LLM was told about.
|
||||
"""
|
||||
transport = self._get_transport()
|
||||
block = _make_tool_use_block("mcp_unknown_tool")
|
||||
response = _make_response(block)
|
||||
|
||||
registry = _FakeRegistry({"read_file"}) # neither name registered
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=True)
|
||||
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "mcp_unknown_tool"
|
||||
|
||||
def test_mixed_tools_same_response(self):
|
||||
"""Both OAuth and native MCP tools in the same response."""
|
||||
transport = self._get_transport()
|
||||
block1 = _make_tool_use_block("mcp_read_file", block_id="tc_1")
|
||||
block2 = _make_tool_use_block("mcp_composio_SEARCH", block_id="tc_2")
|
||||
block3 = _make_tool_use_block("mcp_composio_SEARCH", block_id="tc_3") # also registered natively
|
||||
response = _make_response(block1, block2, block3)
|
||||
|
||||
registry = _FakeRegistry({
|
||||
"read_file", # OAuth-injected
|
||||
"mcp_composio_SEARCH", # native MCP
|
||||
})
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=True)
|
||||
|
||||
assert len(result.tool_calls) == 3
|
||||
# OAuth tool: stripped
|
||||
assert result.tool_calls[0].name == "read_file"
|
||||
# Native MCP: preserved (both stripped and full are registered, full wins)
|
||||
assert result.tool_calls[1].name == "mcp_composio_SEARCH"
|
||||
assert result.tool_calls[2].name == "mcp_composio_SEARCH"
|
||||
|
||||
def test_both_stripped_and_full_registered_prefers_full(self):
|
||||
"""Edge case: both 'foo' and 'mcp_foo' exist in registry.
|
||||
|
||||
Keep 'mcp_foo' (the original name) since it's what the LLM requested.
|
||||
"""
|
||||
transport = self._get_transport()
|
||||
block = _make_tool_use_block("mcp_foo")
|
||||
response = _make_response(block)
|
||||
|
||||
registry = _FakeRegistry({"foo", "mcp_foo"})
|
||||
with patch("tools.registry.registry", registry):
|
||||
result = transport.normalize_response(response, strip_tool_prefix=True)
|
||||
|
||||
assert len(result.tool_calls) == 1
|
||||
# Both exist — the condition `get_entry(stripped) and not get_entry(name)`
|
||||
# is False because get_entry(name) IS truthy, so we keep the full name.
|
||||
assert result.tool_calls[0].name == "mcp_foo"
|
||||
|
||||
|
||||
class TestAnthropicOAuthOutgoingPrefix:
|
||||
"""Verify the outgoing-side companion fix: build_anthropic_kwargs must not
|
||||
double-prefix tool names that already start with ``mcp_`` (native MCP server
|
||||
tools registered as ``mcp_<server>_<tool>``). GH-25255."""
|
||||
|
||||
def _build(self, tools, is_oauth=True):
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
return build_anthropic_kwargs(
|
||||
model="claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
tools=tools,
|
||||
max_tokens=4096,
|
||||
reasoning_config=None,
|
||||
is_oauth=is_oauth,
|
||||
)
|
||||
|
||||
def test_oauth_adds_prefix_to_bare_tool_name(self):
|
||||
"""OAuth + bare name → prefix added (existing Claude Code convention)."""
|
||||
kwargs = self._build([{
|
||||
"type": "function",
|
||||
"function": {"name": "read_file", "description": "x", "parameters": {}},
|
||||
}])
|
||||
names = [t["name"] for t in kwargs["tools"]]
|
||||
assert names == ["mcp_read_file"]
|
||||
|
||||
def test_oauth_does_not_double_prefix_native_mcp_tool(self):
|
||||
"""OAuth + already-prefixed native MCP name → left alone."""
|
||||
kwargs = self._build([{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "mcp_composio_COMPOSIO_SEARCH_TOOLS",
|
||||
"description": "x",
|
||||
"parameters": {},
|
||||
},
|
||||
}])
|
||||
names = [t["name"] for t in kwargs["tools"]]
|
||||
# Must NOT become "mcp_mcp_composio_..." — that breaks the round-trip
|
||||
# because normalize_response only strips ONE mcp_ prefix.
|
||||
assert names == ["mcp_composio_COMPOSIO_SEARCH_TOOLS"]
|
||||
|
||||
def test_oauth_mixed_native_and_bare_tools(self):
|
||||
"""Mixed: native MCP preserved, bare names prefixed."""
|
||||
kwargs = self._build([
|
||||
{"type": "function", "function": {"name": "read_file",
|
||||
"description": "x", "parameters": {}}},
|
||||
{"type": "function", "function": {"name": "mcp_composio_SEARCH",
|
||||
"description": "y", "parameters": {}}},
|
||||
{"type": "function", "function": {"name": "terminal",
|
||||
"description": "z", "parameters": {}}},
|
||||
])
|
||||
names = sorted(t["name"] for t in kwargs["tools"])
|
||||
assert names == ["mcp_composio_SEARCH", "mcp_read_file", "mcp_terminal"]
|
||||
|
||||
def test_non_oauth_path_untouched(self):
|
||||
"""Non-OAuth requests never get the prefix — schemas pass through as-is."""
|
||||
kwargs = self._build([
|
||||
{"type": "function", "function": {"name": "read_file",
|
||||
"description": "x", "parameters": {}}},
|
||||
{"type": "function", "function": {"name": "mcp_composio_SEARCH",
|
||||
"description": "y", "parameters": {}}},
|
||||
], is_oauth=False)
|
||||
names = sorted(t["name"] for t in kwargs["tools"])
|
||||
assert names == ["mcp_composio_SEARCH", "read_file"]
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Regression tests for the Anthropic OAuth PKCE flow.
|
||||
|
||||
Guards against re-introducing the bug where the PKCE ``code_verifier`` was
|
||||
reused as the OAuth ``state`` parameter, leaking the verifier via the
|
||||
authorization URL (browser history, Referer headers, auth-server logs) and
|
||||
removing CSRF protection on the callback path.
|
||||
|
||||
History:
|
||||
- PR #1775 first fixed this on ``run_hermes_oauth_login()``.
|
||||
- PR #2647 (b17e5c10) added ``run_hermes_oauth_login_pure()`` and silently
|
||||
copy-pasted the pre-#1775 vulnerable pattern.
|
||||
- PR #3107 removed the old function, leaving only the regressed copy.
|
||||
- PR #10699 (issue #10693) fixed the regression on the surviving function.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
|
||||
def _patch_oauth_flow(
|
||||
monkeypatch,
|
||||
*,
|
||||
callback_code: str,
|
||||
token_response: Dict[str, Any] | None = None,
|
||||
capture_token_request: Dict[str, Any] | None = None,
|
||||
capture_auth_url: Dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Wire up monkeypatches that let ``run_hermes_oauth_login_pure()`` run
|
||||
end-to-end without touching a real browser, stdin, or HTTP endpoint.
|
||||
|
||||
``callback_code`` is the literal string the user would paste back into the
|
||||
terminal (``"<code>#<state>"`` format).
|
||||
``capture_token_request`` and ``capture_auth_url`` are out-dict captures
|
||||
so the test can introspect what was sent to the auth URL and the token
|
||||
endpoint, respectively.
|
||||
"""
|
||||
import urllib.request
|
||||
|
||||
if token_response is None:
|
||||
token_response = {
|
||||
"access_token": "sk-ant-test-access",
|
||||
"refresh_token": "sk-ant-test-refresh",
|
||||
"expires_in": 3600,
|
||||
}
|
||||
|
||||
def fake_open(url):
|
||||
if capture_auth_url is not None:
|
||||
capture_auth_url["url"] = url
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("webbrowser.open", fake_open)
|
||||
# The flow now gates webbrowser.open() behind a graphical-browser check so
|
||||
# it never launches a console browser (w3m/lynx) inside the terminal. Tests
|
||||
# run headless, so force the GUI path to True — the URL capture relies on
|
||||
# webbrowser.open() being invoked.
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth._can_open_graphical_browser", lambda: True
|
||||
)
|
||||
monkeypatch.setattr("builtins.input", lambda *_a, **_kw: callback_code)
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, body: bytes) -> None:
|
||||
self._body = body
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return self._body
|
||||
|
||||
def fake_urlopen(req, *_a, **_kw):
|
||||
if capture_token_request is not None:
|
||||
capture_token_request["url"] = req.full_url
|
||||
capture_token_request["data"] = json.loads(req.data.decode())
|
||||
capture_token_request["headers"] = dict(req.headers)
|
||||
return _FakeResponse(json.dumps(token_response).encode())
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
|
||||
def test_authorization_url_state_is_not_pkce_verifier(monkeypatch, tmp_path):
|
||||
"""The ``state`` parameter in the authorization URL must NOT equal the
|
||||
PKCE ``code_verifier``.
|
||||
|
||||
Reusing the verifier as state leaks the verifier into browser history,
|
||||
Referer headers, and auth-server access logs — defeating RFC 7636.
|
||||
"""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
captured_url: Dict[str, str] = {}
|
||||
captured_token: Dict[str, Any] = {}
|
||||
_patch_oauth_flow(
|
||||
monkeypatch,
|
||||
# state echoed back unchanged so the CSRF guard passes
|
||||
callback_code="auth-code-from-anthropic#PLACEHOLDER",
|
||||
capture_auth_url=captured_url,
|
||||
capture_token_request=captured_token,
|
||||
)
|
||||
|
||||
# Stub the callback parse: we need the state echoed back to match. To do
|
||||
# that without hardcoding the state value, override input() AFTER seeing
|
||||
# the auth URL.
|
||||
import builtins
|
||||
|
||||
real_input_calls = {"count": 0}
|
||||
|
||||
def fake_input(*_a, **_kw):
|
||||
real_input_calls["count"] += 1
|
||||
# First (and only) call is the "Authorization code:" prompt.
|
||||
url = captured_url.get("url", "")
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
state = qs.get("state", [""])[0]
|
||||
return f"auth-code-from-anthropic#{state}"
|
||||
|
||||
monkeypatch.setattr(builtins, "input", fake_input)
|
||||
|
||||
from agent.anthropic_adapter import run_hermes_oauth_login_pure
|
||||
|
||||
result = run_hermes_oauth_login_pure()
|
||||
assert result is not None, "OAuth flow should succeed with matching state"
|
||||
|
||||
url = captured_url["url"]
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
|
||||
assert "state" in qs and qs["state"][0], "authorization URL must include state"
|
||||
assert "code_challenge" in qs, "authorization URL must include code_challenge"
|
||||
|
||||
state_in_url = qs["state"][0]
|
||||
verifier_sent = captured_token["data"]["code_verifier"]
|
||||
|
||||
# The whole point: state and verifier must be independent values.
|
||||
assert state_in_url != verifier_sent, (
|
||||
"PKCE code_verifier was reused as OAuth state — regression of #10693 / "
|
||||
"#1775. The verifier is supposed to be a secret known only to the "
|
||||
"client; placing it in the authorization URL leaks it via browser "
|
||||
"history, Referer headers, and auth-server logs."
|
||||
)
|
||||
|
||||
# And the verifier MUST NOT appear anywhere in the URL.
|
||||
assert verifier_sent not in url, (
|
||||
"PKCE verifier leaked into authorization URL — regression of #10693"
|
||||
)
|
||||
|
||||
|
||||
def test_callback_state_mismatch_aborts(monkeypatch, tmp_path, caplog):
|
||||
"""If the state returned in the callback does not match the one we sent
|
||||
in the authorization URL, the flow must abort before exchanging the code.
|
||||
|
||||
Without this check, an attacker who tricks the user into pasting a
|
||||
crafted ``<code>#<state>`` string can complete the token exchange — the
|
||||
CSRF protection that ``state`` is supposed to provide (RFC 6749 §10.12)
|
||||
would be absent.
|
||||
"""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
captured_token: Dict[str, Any] = {}
|
||||
_patch_oauth_flow(
|
||||
monkeypatch,
|
||||
callback_code="attacker-code#attacker-state-does-not-match",
|
||||
capture_token_request=captured_token,
|
||||
)
|
||||
|
||||
from agent.anthropic_adapter import run_hermes_oauth_login_pure
|
||||
|
||||
result = run_hermes_oauth_login_pure()
|
||||
|
||||
assert result is None, "mismatched state must abort the flow"
|
||||
assert "url" not in captured_token, (
|
||||
"token exchange must NOT happen when state mismatches"
|
||||
)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Tests for Arcee Trinity Large Thinking per-model overrides.
|
||||
|
||||
Arcee Trinity Large Thinking is a reasoning model that wants:
|
||||
- Fixed temperature=0.5 (vs the global default)
|
||||
- Compression threshold=0.75 (delay compression to preserve reasoning context)
|
||||
|
||||
The helpers must match the bare model name, including when it arrives via
|
||||
OpenRouter as ``arcee-ai/trinity-large-thinking``, but must NOT hit sibling
|
||||
Arcee models like trinity-large-preview or trinity-mini.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.auxiliary_client import (
|
||||
_compression_threshold_for_model,
|
||||
_fixed_temperature_for_model,
|
||||
_is_arcee_trinity_thinking,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"trinity-large-thinking",
|
||||
"arcee-ai/trinity-large-thinking",
|
||||
"Arcee-AI/Trinity-Large-Thinking", # case-insensitive
|
||||
" trinity-large-thinking ", # whitespace tolerant
|
||||
],
|
||||
)
|
||||
def test_is_arcee_trinity_thinking_matches(model: str) -> None:
|
||||
assert _is_arcee_trinity_thinking(model) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
None,
|
||||
"",
|
||||
"trinity-large-preview",
|
||||
"arcee-ai/trinity-large-preview:free",
|
||||
"trinity-mini",
|
||||
"arcee-ai/trinity-mini",
|
||||
"trinity-large", # prefix-only must not match
|
||||
"claude-sonnet-4.6",
|
||||
"gpt-5.4",
|
||||
],
|
||||
)
|
||||
def test_is_arcee_trinity_thinking_rejects_non_matches(model) -> None:
|
||||
assert _is_arcee_trinity_thinking(model) is False
|
||||
|
||||
|
||||
def test_fixed_temperature_for_trinity_thinking() -> None:
|
||||
assert _fixed_temperature_for_model("trinity-large-thinking") == 0.5
|
||||
assert _fixed_temperature_for_model("arcee-ai/trinity-large-thinking") == 0.5
|
||||
|
||||
|
||||
def test_fixed_temperature_sibling_arcee_models_unaffected() -> None:
|
||||
# Preview and mini do not pin temperature — caller chooses its default.
|
||||
assert _fixed_temperature_for_model("trinity-large-preview") is None
|
||||
assert _fixed_temperature_for_model("trinity-mini") is None
|
||||
|
||||
|
||||
def test_compression_threshold_for_trinity_thinking() -> None:
|
||||
assert _compression_threshold_for_model("trinity-large-thinking") == 0.75
|
||||
assert _compression_threshold_for_model("arcee-ai/trinity-large-thinking") == 0.75
|
||||
|
||||
|
||||
def test_compression_threshold_default_none_for_other_models() -> None:
|
||||
# None means "leave the user's config value unchanged".
|
||||
assert _compression_threshold_for_model(None) is None
|
||||
assert _compression_threshold_for_model("") is None
|
||||
assert _compression_threshold_for_model("trinity-large-preview") is None
|
||||
assert _compression_threshold_for_model("claude-sonnet-4.6") is None
|
||||
assert _compression_threshold_for_model("kimi-k2") is None
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tests for agent.async_utils.safe_schedule_threadsafe."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import warnings
|
||||
from concurrent.futures import Future
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
from agent.async_utils import safe_schedule_threadsafe
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _no_unawaited_warnings(caught, *, coro_name: str = "") -> bool:
|
||||
"""Return True if no "X was never awaited" warning slipped through.
|
||||
|
||||
When *coro_name* is provided, only warnings naming that coroutine are
|
||||
counted — xdist workers may emit unrelated unawaited-coroutine warnings
|
||||
(e.g. ``AsyncMockMixin._execute_mock_call``) from concurrent tests.
|
||||
"""
|
||||
bad = [
|
||||
w for w in caught
|
||||
if issubclass(w.category, RuntimeWarning)
|
||||
and "was never awaited" in str(w.message)
|
||||
and (not coro_name or coro_name in str(w.message))
|
||||
]
|
||||
return not bad
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSafeScheduleThreadsafe:
|
||||
def test_returns_future_on_success(self):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
import threading
|
||||
ready = threading.Event()
|
||||
stop = threading.Event()
|
||||
|
||||
def _runner():
|
||||
asyncio.set_event_loop(loop)
|
||||
ready.set()
|
||||
loop.run_until_complete(_wait_for_stop(stop))
|
||||
|
||||
async def _wait_for_stop(ev):
|
||||
while not ev.is_set():
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
t = threading.Thread(target=_runner, daemon=True)
|
||||
t.start()
|
||||
ready.wait(timeout=2)
|
||||
|
||||
async def _sample():
|
||||
return 42
|
||||
|
||||
fut = safe_schedule_threadsafe(_sample(), loop)
|
||||
assert isinstance(fut, Future)
|
||||
assert fut.result(timeout=2) == 42
|
||||
|
||||
stop.set()
|
||||
t.join(timeout=2)
|
||||
finally:
|
||||
if loop.is_running():
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
loop.close()
|
||||
|
||||
def test_closed_loop_returns_none_and_closes_coroutine(self):
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.close()
|
||||
|
||||
async def _sample():
|
||||
return "ok"
|
||||
|
||||
coro = _sample()
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
result = safe_schedule_threadsafe(coro, loop)
|
||||
del coro
|
||||
gc.collect()
|
||||
|
||||
assert result is None
|
||||
assert _no_unawaited_warnings(caught, coro_name='_sample')
|
||||
|
||||
def test_none_loop_returns_none_and_closes_coroutine(self):
|
||||
async def _sample():
|
||||
return "ok"
|
||||
|
||||
coro = _sample()
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
result = safe_schedule_threadsafe(coro, None)
|
||||
del coro
|
||||
gc.collect()
|
||||
|
||||
assert result is None
|
||||
assert _no_unawaited_warnings(caught, coro_name='_sample')
|
||||
|
||||
def test_scheduling_exception_closes_coroutine(self):
|
||||
"""If run_coroutine_threadsafe raises, close the coroutine and return None."""
|
||||
# A loop that *looks* open but raises on submission
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
async def _sample():
|
||||
return "ok"
|
||||
|
||||
coro = _sample()
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
with patch(
|
||||
"agent.async_utils.asyncio.run_coroutine_threadsafe",
|
||||
side_effect=RuntimeError("scheduler down"),
|
||||
):
|
||||
result = safe_schedule_threadsafe(coro, loop)
|
||||
del coro
|
||||
gc.collect()
|
||||
|
||||
assert result is None
|
||||
assert _no_unawaited_warnings(caught, coro_name='_sample')
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
def test_logs_at_specified_level(self, caplog):
|
||||
import logging
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.close()
|
||||
|
||||
async def _sample():
|
||||
return None
|
||||
|
||||
custom = logging.getLogger("test_async_utils")
|
||||
with caplog.at_level(logging.WARNING, logger="test_async_utils"):
|
||||
result = safe_schedule_threadsafe(
|
||||
_sample(), loop,
|
||||
logger=custom,
|
||||
log_message="custom-msg",
|
||||
log_level=logging.WARNING,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert any("custom-msg" in rec.message for rec in caplog.records)
|
||||
|
||||
def test_non_coroutine_arg_does_not_crash(self):
|
||||
"""Defensive: even if the caller hands us something weird, don't blow up."""
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.close()
|
||||
|
||||
# Pass a non-coroutine sentinel
|
||||
result = safe_schedule_threadsafe("not-a-coroutine", loop) # type: ignore[arg-type]
|
||||
assert result is None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
"""Tests for agent.auxiliary_client._try_custom_endpoint's anthropic_messages branch.
|
||||
|
||||
When a user configures a custom endpoint with ``api_mode: anthropic_messages``
|
||||
(e.g. MiniMax, Zhipu GLM, LiteLLM in Anthropic-proxy mode), auxiliary tasks
|
||||
(compression, web_extract, session_search, title generation) must use the
|
||||
native Anthropic transport rather than being silently downgraded to an
|
||||
OpenAI-wire client that speaks the wrong protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch):
|
||||
for key in (
|
||||
"OPENAI_API_KEY", "OPENAI_BASE_URL",
|
||||
"ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
|
||||
def _install_anthropic_adapter_mocks():
|
||||
"""Patch build_anthropic_client so the test doesn't need the SDK."""
|
||||
fake_client = MagicMock(name="anthropic_client")
|
||||
return patch(
|
||||
"agent.anthropic_adapter.build_anthropic_client",
|
||||
return_value=fake_client,
|
||||
), fake_client
|
||||
|
||||
|
||||
def test_custom_endpoint_anthropic_messages_builds_anthropic_wrapper():
|
||||
"""api_mode=anthropic_messages → returns AnthropicAuxiliaryClient, not OpenAI."""
|
||||
from agent.auxiliary_client import _try_custom_endpoint, AnthropicAuxiliaryClient
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_custom_runtime",
|
||||
return_value=(
|
||||
"https://api.minimax.io/anthropic",
|
||||
"minimax-key",
|
||||
"anthropic_messages",
|
||||
),
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="claude-sonnet-4-6",
|
||||
):
|
||||
adapter_patch, fake_client = _install_anthropic_adapter_mocks()
|
||||
with adapter_patch:
|
||||
client, model = _try_custom_endpoint()
|
||||
|
||||
assert isinstance(client, AnthropicAuxiliaryClient), (
|
||||
"Custom endpoint with api_mode=anthropic_messages must return the "
|
||||
f"native Anthropic wrapper, got {type(client).__name__}"
|
||||
)
|
||||
assert model == "claude-sonnet-4-6"
|
||||
# Wrapper should NOT be marked as OAuth — third-party endpoints are
|
||||
# always API-key authenticated.
|
||||
assert client.api_key == "minimax-key"
|
||||
assert client.base_url == "https://api.minimax.io/anthropic"
|
||||
|
||||
|
||||
def test_custom_endpoint_anthropic_messages_falls_back_when_sdk_missing():
|
||||
"""Graceful degradation when anthropic SDK is unavailable."""
|
||||
from agent.auxiliary_client import _try_custom_endpoint
|
||||
|
||||
import_error = ImportError("anthropic package not installed")
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_custom_runtime",
|
||||
return_value=("https://api.minimax.io/anthropic", "k", "anthropic_messages"),
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="claude-sonnet-4-6",
|
||||
), patch(
|
||||
"agent.anthropic_adapter.build_anthropic_client",
|
||||
side_effect=import_error,
|
||||
):
|
||||
client, model = _try_custom_endpoint()
|
||||
|
||||
# Should fall back to an OpenAI-wire client rather than returning
|
||||
# (None, None) — the tool still needs to do *something*.
|
||||
assert client is not None
|
||||
assert model == "claude-sonnet-4-6"
|
||||
# OpenAI client, not AnthropicAuxiliaryClient.
|
||||
from agent.auxiliary_client import AnthropicAuxiliaryClient
|
||||
assert not isinstance(client, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_custom_endpoint_chat_completions_still_uses_openai_wire():
|
||||
"""Regression: default path (no api_mode) must remain OpenAI client."""
|
||||
from agent.auxiliary_client import _try_custom_endpoint, AnthropicAuxiliaryClient
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_custom_runtime",
|
||||
return_value=("https://api.example.com/v1", "key", None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="my-model",
|
||||
):
|
||||
client, model = _try_custom_endpoint()
|
||||
|
||||
assert client is not None
|
||||
assert model == "my-model"
|
||||
assert not isinstance(client, AnthropicAuxiliaryClient)
|
||||
@@ -0,0 +1,349 @@
|
||||
"""Tests for auxiliary client routing of the ``azure-foundry`` provider.
|
||||
|
||||
Covers the dedicated branch in ``agent.auxiliary_client.resolve_provider_client``
|
||||
that delegates to :func:`hermes_cli.runtime_provider._resolve_azure_foundry_runtime`
|
||||
instead of falling into the generic ``resolve_api_key_provider_credentials``
|
||||
path (which only knows about ``AZURE_FOUNDRY_API_KEY`` and would 401 for
|
||||
Entra ID users and miss ``model.base_url`` overrides for api-key users
|
||||
with non-standard Foundry-projects endpoints).
|
||||
|
||||
Pinned scenarios:
|
||||
|
||||
* ``auth_mode: api_key`` → plain OpenAI client with the static string
|
||||
key for ``chat_completions``.
|
||||
* ``auth_mode: entra_id`` + ``chat_completions`` → plain OpenAI
|
||||
client with a callable ``api_key`` (the bearer-token provider) —
|
||||
confirms the callable survives the auxiliary path end-to-end.
|
||||
* ``auth_mode: entra_id`` + GPT-5.x model → CodexAuxiliaryClient
|
||||
wrapping the OpenAI client (api_mode auto-upgrades to
|
||||
codex_responses).
|
||||
* Anthropic-style + entra_id → rejected at the runtime resolver,
|
||||
so the aux path returns ``(None, None)``.
|
||||
* Failure path when no model is configured returns ``(None, None)``
|
||||
cleanly so the auto chain falls through.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_credential_cache():
|
||||
from agent.azure_identity_adapter import reset_credential_cache
|
||||
reset_credential_cache()
|
||||
yield
|
||||
reset_credential_cache()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_azure_identity(monkeypatch):
|
||||
"""Stand-in for azure.identity (keeps CI hermetic when the SDK is
|
||||
not installed)."""
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
|
||||
last = {"scope": None}
|
||||
|
||||
def _provider(scope):
|
||||
return lambda: f"jwt-for-{scope}"
|
||||
|
||||
fake_module = SimpleNamespace(
|
||||
DefaultAzureCredential=lambda **kw: SimpleNamespace(
|
||||
kwargs=kw,
|
||||
get_token=lambda scope: SimpleNamespace(token="fake", expires_on=9999999999),
|
||||
),
|
||||
get_bearer_token_provider=lambda credential, scope: (
|
||||
last.__setitem__("scope", scope),
|
||||
_provider(scope),
|
||||
)[-1],
|
||||
)
|
||||
monkeypatch.setattr(_adapter, "_require_azure_identity", lambda: fake_module)
|
||||
monkeypatch.setitem(sys.modules, "azure.identity", fake_module)
|
||||
return last
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patch_load_config(monkeypatch):
|
||||
"""Helper to set model_cfg seen by _try_azure_foundry."""
|
||||
def _apply(model_cfg):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.load_config",
|
||||
lambda: {"model": model_cfg},
|
||||
)
|
||||
return _apply
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# auth_mode: api_key (default) — regression for the legacy path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAuxAzureFoundryApiKey:
|
||||
def test_chat_completions_returns_plain_openai_client(self, monkeypatch, patch_load_config):
|
||||
from agent.auxiliary_client import _try_azure_foundry
|
||||
from openai import OpenAI as _OpenAI
|
||||
|
||||
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "sk-azure-static-key")
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
"default": "gpt-4o",
|
||||
})
|
||||
client, resolved = _try_azure_foundry(model="gpt-4o")
|
||||
assert client is not None
|
||||
assert resolved == "gpt-4o"
|
||||
assert isinstance(client, _OpenAI)
|
||||
assert client.api_key == "sk-azure-static-key"
|
||||
|
||||
def test_codex_responses_wraps_in_codex_aux_client(self, monkeypatch, patch_load_config):
|
||||
from agent.auxiliary_client import _try_azure_foundry, CodexAuxiliaryClient
|
||||
|
||||
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "sk-azure-static-key")
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
"default": "gpt-5.4-mini",
|
||||
})
|
||||
# GPT-5.x → runtime auto-upgrades to codex_responses
|
||||
client, resolved = _try_azure_foundry(model="gpt-5.4-mini")
|
||||
assert resolved == "gpt-5.4-mini"
|
||||
assert isinstance(client, CodexAuxiliaryClient)
|
||||
assert client.api_key == "sk-azure-static-key"
|
||||
|
||||
def test_no_key_returns_none(self, monkeypatch, patch_load_config):
|
||||
from agent.auxiliary_client import _try_azure_foundry
|
||||
|
||||
monkeypatch.delenv("AZURE_FOUNDRY_API_KEY", raising=False)
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
"default": "gpt-4o",
|
||||
})
|
||||
client, resolved = _try_azure_foundry(model="gpt-4o")
|
||||
assert client is None
|
||||
assert resolved is None
|
||||
|
||||
def test_no_model_returns_none(self, monkeypatch, patch_load_config):
|
||||
"""Azure has no fallback aux model — fail soft so the auto chain
|
||||
can try other providers."""
|
||||
from agent.auxiliary_client import _try_azure_foundry
|
||||
|
||||
monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "sk-azure-static-key")
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
# No default model
|
||||
})
|
||||
client, resolved = _try_azure_foundry()
|
||||
assert client is None
|
||||
assert resolved is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# auth_mode: entra_id — callable api_key survives end-to-end
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAuxAzureFoundryEntra:
|
||||
def test_callable_api_key_reaches_openai_constructor(
|
||||
self, monkeypatch, fake_azure_identity, patch_load_config,
|
||||
):
|
||||
"""The token provider callable must arrive at ``OpenAI(api_key=...)``
|
||||
intact — never stringified to ``"no-key-required"`` or to the
|
||||
SDK-internal empty-string representation BEFORE we hand it off.
|
||||
|
||||
We assert on the public SDK contract (constructor receives the
|
||||
callable) rather than ``client.api_key``, because OpenAI 2.24.0
|
||||
stores callable api_keys in a private attribute and exposes
|
||||
``client.api_key`` as ``""``. The SDK still calls the callable
|
||||
per request to mint ``Authorization: Bearer <token>``; that
|
||||
behaviour is the documented Microsoft/OpenAI contract we rely on.
|
||||
"""
|
||||
from agent import auxiliary_client as _aux
|
||||
|
||||
received = {}
|
||||
|
||||
class _FakeOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
received.update(kwargs)
|
||||
# Mirror the fields downstream callers read.
|
||||
self.api_key = kwargs.get("api_key", "")
|
||||
self.base_url = kwargs.get("base_url", "")
|
||||
|
||||
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
"auth_mode": "entra_id",
|
||||
"default": "gpt-4o",
|
||||
})
|
||||
client, resolved = _aux._try_azure_foundry(model="gpt-4o")
|
||||
assert client is not None
|
||||
assert resolved == "gpt-4o"
|
||||
# Public-contract assertion: the OpenAI SDK constructor saw the
|
||||
# callable, exactly as Microsoft's Foundry sample requires.
|
||||
assert callable(received["api_key"])
|
||||
assert not isinstance(received["api_key"], str)
|
||||
assert received["api_key"]().startswith("jwt-for-")
|
||||
# Base URL forwarded verbatim (no /responses suffix stripping
|
||||
# in this path — that's a separate concern handled by the
|
||||
# runtime resolver only when the user re-saves config).
|
||||
assert received["base_url"] == "https://r.openai.azure.com/openai/v1"
|
||||
|
||||
def test_codex_responses_with_entra_wraps_correctly(
|
||||
self, monkeypatch, fake_azure_identity, patch_load_config,
|
||||
):
|
||||
"""GPT-5.x deployment on Entra ID — auto-upgraded to
|
||||
codex_responses, wrapped in CodexAuxiliaryClient, callable
|
||||
api_key handed to the underlying OpenAI SDK."""
|
||||
from agent import auxiliary_client as _aux
|
||||
|
||||
received = {}
|
||||
|
||||
class _FakeOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
received.update(kwargs)
|
||||
self.api_key = kwargs.get("api_key", "")
|
||||
self.base_url = kwargs.get("base_url", "")
|
||||
|
||||
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
"auth_mode": "entra_id",
|
||||
"default": "gpt-5.4-mini",
|
||||
})
|
||||
client, resolved = _aux._try_azure_foundry(model="gpt-5.4-mini")
|
||||
assert resolved == "gpt-5.4-mini"
|
||||
assert isinstance(client, _aux.CodexAuxiliaryClient)
|
||||
# The Codex wrapper received an OpenAI client built with the
|
||||
# callable api_key — verify against the SDK constructor record,
|
||||
# not the wrapper attribute (which mirrors the SDK's empty-
|
||||
# string representation).
|
||||
assert callable(received["api_key"])
|
||||
assert received["api_key"]().startswith("jwt-for-")
|
||||
|
||||
def test_entra_anthropic_messages_uses_bearer_hook(
|
||||
self, monkeypatch, fake_azure_identity, patch_load_config,
|
||||
):
|
||||
"""Entra ID + anthropic_messages: runtime returns a callable
|
||||
api_key; ``_maybe_wrap_anthropic`` → ``build_anthropic_client``
|
||||
detects the callable and installs the bearer-injecting httpx
|
||||
event hook on a custom ``httpx.Client`` passed to the
|
||||
Anthropic SDK via ``http_client=``."""
|
||||
from agent import auxiliary_client as _aux
|
||||
from agent import anthropic_adapter as _anthropic
|
||||
|
||||
received = {}
|
||||
|
||||
class _FakeOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
received["openai"] = kwargs
|
||||
self.api_key = kwargs.get("api_key", "")
|
||||
self.base_url = kwargs.get("base_url", "")
|
||||
|
||||
class _FakeAnthropicSDK:
|
||||
class Anthropic:
|
||||
def __init__(self, **kwargs):
|
||||
received["anthropic"] = kwargs
|
||||
|
||||
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
|
||||
monkeypatch.setattr(_anthropic, "_get_anthropic_sdk", lambda: _FakeAnthropicSDK)
|
||||
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.services.ai.azure.com/anthropic",
|
||||
"api_mode": "anthropic_messages",
|
||||
"auth_mode": "entra_id",
|
||||
"default": "claude-sonnet-4-5",
|
||||
})
|
||||
client, resolved = _aux._try_azure_foundry(model="claude-sonnet-4-5")
|
||||
assert client is not None
|
||||
assert resolved == "claude-sonnet-4-5"
|
||||
# The Anthropic SDK constructor received a custom http_client
|
||||
# (the bearer-injecting hook) and a placeholder auth_token.
|
||||
anthropic_kwargs = received.get("anthropic") or {}
|
||||
assert "http_client" in anthropic_kwargs, (
|
||||
"build_anthropic_client must pass a custom http_client when "
|
||||
"given a callable api_key, otherwise the SDK cannot mint "
|
||||
"fresh tokens per request"
|
||||
)
|
||||
assert anthropic_kwargs.get("auth_token") == "entra-id-bearer-via-http-hook"
|
||||
# Verify the http_client actually has our event hook installed.
|
||||
http_client = anthropic_kwargs["http_client"]
|
||||
hooks = getattr(http_client, "event_hooks", {})
|
||||
assert "request" in hooks and len(hooks["request"]) >= 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_provider_client → azure-foundry dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveProviderClientAzureFoundry:
|
||||
def test_dispatches_to_azure_branch_not_generic_api_key_path(
|
||||
self, monkeypatch, fake_azure_identity, patch_load_config,
|
||||
):
|
||||
"""End-to-end: the public ``resolve_provider_client`` entry
|
||||
point must take the dedicated azure-foundry branch, NOT the
|
||||
generic api-key registry path that would call
|
||||
``resolve_api_key_provider_credentials`` and return None for
|
||||
Entra users."""
|
||||
from agent import auxiliary_client as _aux
|
||||
|
||||
received = {}
|
||||
|
||||
class _FakeOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
received.update(kwargs)
|
||||
self.api_key = kwargs.get("api_key", "")
|
||||
self.base_url = kwargs.get("base_url", "")
|
||||
|
||||
monkeypatch.setattr(_aux, "OpenAI", _FakeOpenAI)
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
"auth_mode": "entra_id",
|
||||
"default": "gpt-4o",
|
||||
})
|
||||
client, resolved = _aux.resolve_provider_client("azure-foundry", "gpt-4o")
|
||||
assert client is not None
|
||||
assert resolved == "gpt-4o"
|
||||
# The callable made it through resolve_provider_client → _try_azure_foundry
|
||||
# → OpenAI(api_key=...).
|
||||
assert callable(received["api_key"])
|
||||
|
||||
def test_warns_and_returns_none_on_failure(
|
||||
self, monkeypatch, patch_load_config, caplog,
|
||||
):
|
||||
"""When azure-foundry is requested but cannot be resolved
|
||||
(e.g. no model + no key), we return (None, None) and log a
|
||||
clear warning pointing at ``hermes doctor``."""
|
||||
import logging
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
|
||||
monkeypatch.delenv("AZURE_FOUNDRY_API_KEY", raising=False)
|
||||
patch_load_config({
|
||||
"provider": "azure-foundry",
|
||||
"base_url": "https://r.openai.azure.com/openai/v1",
|
||||
"api_mode": "chat_completions",
|
||||
# No default → resolver yields no model → bail
|
||||
})
|
||||
with caplog.at_level(logging.WARNING, logger="agent.auxiliary_client"):
|
||||
client, resolved = resolve_provider_client("azure-foundry")
|
||||
assert client is None
|
||||
assert resolved is None
|
||||
assert any(
|
||||
"azure-foundry" in rec.message and "hermes doctor" in rec.message
|
||||
for rec in caplog.records
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Tests for xAI OAuth 403 error recovery in auxiliary_client.
|
||||
|
||||
xAI returns HTTP 403 (not 401) with "unauthenticated:bad-credentials" when
|
||||
an OAuth2 access token has expired. These tests verify the three fixes:
|
||||
|
||||
1. _is_auth_error detects xAI 403 as an auth failure
|
||||
2. _recoverable_pool_provider maps api.x.ai to xai-oauth
|
||||
3. _refresh_provider_credentials includes xai-oauth refresh logic
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ── _is_auth_error ──────────────────────────────────────────────────────────
|
||||
|
||||
def _import_is_auth_error():
|
||||
from agent.auxiliary_client import _is_auth_error
|
||||
return _is_auth_error
|
||||
|
||||
|
||||
class TestIsAuthErrorXaiOauth403:
|
||||
"""Verify _is_auth_error correctly identifies xAI's 403 bad-credentials."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _import(self):
|
||||
self.is_auth_error = _import_is_auth_error()
|
||||
|
||||
def test_xai_403_bad_credentials_is_auth_error(self):
|
||||
"""The exact error xAI returns for expired OAuth tokens."""
|
||||
exc = Exception(
|
||||
"Error code: 403 - {'code': 'The caller does not have permission "
|
||||
"to execute the specified operation', 'error': 'The OAuth2 access "
|
||||
"token could not be validated. [WKE=unauthenticated:bad-credentials]'}"
|
||||
)
|
||||
exc.status_code = 403 # openai.PermissionDenied sets this
|
||||
assert self.is_auth_error(exc) is True
|
||||
|
||||
def test_xai_403_bad_credentials_without_status_code(self):
|
||||
"""Fallback match when status_code attribute is missing."""
|
||||
exc = Exception(
|
||||
"Error code: 403 - unauthenticated:bad-credentials"
|
||||
)
|
||||
# No status_code attribute — should still match via string pattern
|
||||
assert self.is_auth_error(exc) is True
|
||||
|
||||
def test_generic_403_is_not_auth_error(self):
|
||||
"""A generic 403 (e.g. rate limit, forbidden) should NOT be treated as auth."""
|
||||
exc = Exception("Error code: 403 - rate limit exceeded")
|
||||
exc.status_code = 403
|
||||
assert self.is_auth_error(exc) is False
|
||||
|
||||
def test_401_status_code_is_auth_error(self):
|
||||
"""Existing 401 detection still works."""
|
||||
exc = Exception("Unauthorized")
|
||||
exc.status_code = 401
|
||||
assert self.is_auth_error(exc) is True
|
||||
|
||||
def test_401_string_is_auth_error(self):
|
||||
"""Existing string-based 401 detection still works."""
|
||||
exc = Exception("Error code: 401 - Unauthorized")
|
||||
assert self.is_auth_error(exc) is True
|
||||
|
||||
def test_authentication_error_class_is_auth_error(self):
|
||||
"""Existing AuthenticationError class detection still works."""
|
||||
exc_type = type("AuthenticationError", (Exception,), {})
|
||||
exc = exc_type("auth failure")
|
||||
assert self.is_auth_error(exc) is True
|
||||
|
||||
def test_permission_denied_without_bad_credentials_is_not_auth_error(self):
|
||||
"""403 PermissionDenied without bad-credentials should not be auth."""
|
||||
exc = Exception("Error code: 403 - Permission denied")
|
||||
exc.status_code = 403
|
||||
assert self.is_auth_error(exc) is False
|
||||
|
||||
def test_500_is_not_auth_error(self):
|
||||
"""Server errors are not auth errors."""
|
||||
exc = Exception("Error code: 500 - Internal server error")
|
||||
exc.status_code = 500
|
||||
assert self.is_auth_error(exc) is False
|
||||
|
||||
def test_unauthenticated_without_bad_credentials_is_not_auth_error(self):
|
||||
"""'unauthenticated' alone (without 'bad-credentials') should not match."""
|
||||
exc = Exception("unauthenticated request")
|
||||
assert self.is_auth_error(exc) is False
|
||||
|
||||
|
||||
# ── _recoverable_pool_provider ──────────────────────────────────────────────
|
||||
|
||||
def _import_recoverable_pool_provider():
|
||||
from agent.auxiliary_client import _recoverable_pool_provider
|
||||
return _recoverable_pool_provider
|
||||
|
||||
|
||||
class TestRecoverablePoolProviderXaiOAuth:
|
||||
"""Verify _recoverable_pool_provider maps api.x.ai to xai-oauth."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _import(self):
|
||||
self.recover = _import_recoverable_pool_provider()
|
||||
|
||||
def test_explicit_xai_oauth_provider(self):
|
||||
"""Explicit provider name passes through."""
|
||||
result = self.recover("xai-oauth", None)
|
||||
assert result == "xai-oauth"
|
||||
|
||||
def test_api_x_ai_host_match(self):
|
||||
"""api.x.ai base URL maps to xai-oauth pool."""
|
||||
class MockClient:
|
||||
base_url = "https://api.x.ai/v1/"
|
||||
|
||||
result = self.recover("auto", MockClient())
|
||||
assert result == "xai-oauth"
|
||||
|
||||
def test_auto_with_unknown_host_returns_none(self):
|
||||
"""auto provider with unknown host returns None."""
|
||||
class MockClient:
|
||||
base_url = "https://unknown.example.com/v1/"
|
||||
|
||||
result = self.recover("auto", MockClient())
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── _refresh_provider_credentials (structure check) ─────────────────────────
|
||||
|
||||
def _import_refresh_provider_credentials():
|
||||
from agent.auxiliary_client import _refresh_provider_credentials
|
||||
return _refresh_provider_credentials
|
||||
|
||||
|
||||
class TestRefreshProviderCredentialsXaiOAuth:
|
||||
"""Verify _refresh_provider_credentials has xai-oauth branch.
|
||||
|
||||
Full integration testing requires live OAuth tokens, so we verify
|
||||
the branch exists and handles the no-credential case gracefully.
|
||||
"""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _import(self):
|
||||
self.refresh = _import_refresh_provider_credentials()
|
||||
|
||||
def test_xai_oauth_no_pool_returns_false(self):
|
||||
"""When no xai-oauth pool exists, refresh returns False gracefully."""
|
||||
# This tests that the branch exists and doesn't crash.
|
||||
# It may return True if the singleton resolver finds tokens,
|
||||
# or False if neither pool nor singleton has credentials.
|
||||
# Either way, it should not raise an exception.
|
||||
result = self.refresh("xai-oauth")
|
||||
assert isinstance(result, bool)
|
||||
|
||||
def test_unknown_provider_returns_false(self):
|
||||
"""Unknown providers fall through to return False."""
|
||||
result = self.refresh("unknown-provider-xyz")
|
||||
assert result is False
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Tests for auxiliary model config bridging — verifies that config.yaml values
|
||||
are properly mapped to environment variables by both CLI and gateway loaders.
|
||||
|
||||
Also tests the vision_tools and browser_tool model override env vars.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
|
||||
|
||||
def _run_auxiliary_bridge(config_dict, monkeypatch):
|
||||
"""Simulate the auxiliary config → env var bridging logic shared by CLI and gateway.
|
||||
|
||||
This mirrors the code in cli.py load_cli_config() and gateway/run.py.
|
||||
Both use the same pattern; we test it once here.
|
||||
"""
|
||||
# Clear env vars
|
||||
for key in (
|
||||
"AUXILIARY_VISION_PROVIDER", "AUXILIARY_VISION_MODEL",
|
||||
"AUXILIARY_VISION_BASE_URL", "AUXILIARY_VISION_API_KEY",
|
||||
"AUXILIARY_WEB_EXTRACT_PROVIDER", "AUXILIARY_WEB_EXTRACT_MODEL",
|
||||
"AUXILIARY_WEB_EXTRACT_BASE_URL", "AUXILIARY_WEB_EXTRACT_API_KEY",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
# Compression config is read directly from config.yaml — no env var bridging.
|
||||
|
||||
# Auxiliary bridge
|
||||
auxiliary_cfg = config_dict.get("auxiliary", {})
|
||||
if auxiliary_cfg and isinstance(auxiliary_cfg, dict):
|
||||
aux_task_env = {
|
||||
"vision": {
|
||||
"provider": "AUXILIARY_VISION_PROVIDER",
|
||||
"model": "AUXILIARY_VISION_MODEL",
|
||||
"base_url": "AUXILIARY_VISION_BASE_URL",
|
||||
"api_key": "AUXILIARY_VISION_API_KEY",
|
||||
},
|
||||
"web_extract": {
|
||||
"provider": "AUXILIARY_WEB_EXTRACT_PROVIDER",
|
||||
"model": "AUXILIARY_WEB_EXTRACT_MODEL",
|
||||
"base_url": "AUXILIARY_WEB_EXTRACT_BASE_URL",
|
||||
"api_key": "AUXILIARY_WEB_EXTRACT_API_KEY",
|
||||
},
|
||||
}
|
||||
for task_key, env_map in aux_task_env.items():
|
||||
task_cfg = auxiliary_cfg.get(task_key, {})
|
||||
if not isinstance(task_cfg, dict):
|
||||
continue
|
||||
prov = str(task_cfg.get("provider", "")).strip()
|
||||
model = str(task_cfg.get("model", "")).strip()
|
||||
base_url = str(task_cfg.get("base_url", "")).strip()
|
||||
api_key = str(task_cfg.get("api_key", "")).strip()
|
||||
if prov and prov != "auto":
|
||||
os.environ[env_map["provider"]] = prov
|
||||
if model:
|
||||
os.environ[env_map["model"]] = model
|
||||
if base_url:
|
||||
os.environ[env_map["base_url"]] = base_url
|
||||
if api_key:
|
||||
os.environ[env_map["api_key"]] = api_key
|
||||
|
||||
|
||||
# ── Config bridging tests ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestAuxiliaryConfigBridge:
|
||||
"""Verify the config.yaml → env var bridging logic used by CLI and gateway."""
|
||||
|
||||
def test_vision_provider_bridged(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {"provider": "openrouter", "model": ""},
|
||||
"web_extract": {"provider": "auto", "model": ""},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
|
||||
# auto should not be set
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None
|
||||
|
||||
def test_vision_model_bridged(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {"provider": "auto", "model": "openai/gpt-4o"},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_MODEL") == "openai/gpt-4o"
|
||||
# auto provider should not be set
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
|
||||
|
||||
def test_web_extract_bridged(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"web_extract": {"provider": "nous", "model": "gemini-2.5-flash"},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") == "nous"
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "gemini-2.5-flash"
|
||||
|
||||
def test_direct_endpoint_bridged(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {
|
||||
"base_url": "http://localhost:1234/v1",
|
||||
"api_key": "local-key",
|
||||
"model": "qwen2.5-vl",
|
||||
}
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_BASE_URL") == "http://localhost:1234/v1"
|
||||
assert os.environ.get("AUXILIARY_VISION_API_KEY") == "local-key"
|
||||
assert os.environ.get("AUXILIARY_VISION_MODEL") == "qwen2.5-vl"
|
||||
|
||||
def test_empty_values_not_bridged(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {"provider": "auto", "model": ""},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
|
||||
assert os.environ.get("AUXILIARY_VISION_MODEL") is None
|
||||
|
||||
def test_missing_auxiliary_section_safe(self, monkeypatch):
|
||||
"""Config without auxiliary section should not crash."""
|
||||
config = {"model": {"default": "test-model"}}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
|
||||
|
||||
def test_non_dict_task_config_ignored(self, monkeypatch):
|
||||
"""Malformed task config (e.g. string instead of dict) is safely ignored."""
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": "openrouter", # should be a dict
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
|
||||
|
||||
def test_mixed_tasks(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {"provider": "openrouter", "model": ""},
|
||||
"web_extract": {"provider": "auto", "model": "custom-llm"},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
|
||||
assert os.environ.get("AUXILIARY_VISION_MODEL") is None
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "custom-llm"
|
||||
|
||||
def test_all_tasks_with_overrides(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"},
|
||||
"web_extract": {"provider": "nous", "model": "gemini-3-flash"},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
|
||||
assert os.environ.get("AUXILIARY_VISION_MODEL") == "google/gemini-2.5-flash"
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") == "nous"
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "gemini-3-flash"
|
||||
|
||||
def test_whitespace_in_values_stripped(self, monkeypatch):
|
||||
config = {
|
||||
"auxiliary": {
|
||||
"vision": {"provider": " openrouter ", "model": " my-model "},
|
||||
}
|
||||
}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter"
|
||||
assert os.environ.get("AUXILIARY_VISION_MODEL") == "my-model"
|
||||
|
||||
def test_empty_auxiliary_dict_safe(self, monkeypatch):
|
||||
config = {"auxiliary": {}}
|
||||
_run_auxiliary_bridge(config, monkeypatch)
|
||||
assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None
|
||||
assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None
|
||||
|
||||
|
||||
# ── Gateway bridge parity test ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGatewayBridgeCodeParity:
|
||||
"""Verify the gateway/run.py config bridge contains the auxiliary section."""
|
||||
|
||||
def test_gateway_has_auxiliary_bridge(self):
|
||||
"""The gateway config bridge must include auxiliary.* bridging.
|
||||
|
||||
After the plugin-aux-task API refactor (2026-05), gateway env-var
|
||||
names are derived dynamically (``AUXILIARY_<KEY_UPPER>_*``) so the
|
||||
literal strings ``AUXILIARY_VISION_PROVIDER`` etc. no longer appear
|
||||
in source. Assert the dynamic shape and the canonical built-in keys
|
||||
bridged set instead.
|
||||
"""
|
||||
gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py"
|
||||
# Pin encoding to UTF-8: source files in this repo are UTF-8, but
|
||||
# Path.read_text() defaults to the system locale — which is cp1252
|
||||
# on most Western Windows installs and crashes as soon as the file
|
||||
# contains any non-ASCII byte (e.g. an em-dash in a comment).
|
||||
content = gateway_path.read_text(encoding="utf-8")
|
||||
# Dynamic env-var derivation present
|
||||
assert 'f"AUXILIARY_{_upper}_PROVIDER"' in content
|
||||
assert 'f"AUXILIARY_{_upper}_MODEL"' in content
|
||||
assert 'f"AUXILIARY_{_upper}_BASE_URL"' in content
|
||||
assert 'f"AUXILIARY_{_upper}_API_KEY"' in content
|
||||
# Built-in bridged keys present
|
||||
assert "_aux_bridged_keys" in content
|
||||
assert '"vision"' in content
|
||||
assert '"web_extract"' in content
|
||||
assert '"approval"' in content
|
||||
# Plugin-aux-task discovery hooked into bridging
|
||||
assert "get_plugin_auxiliary_tasks" in content
|
||||
|
||||
def test_gateway_no_compression_env_bridge(self):
|
||||
"""Gateway should NOT bridge compression config to env vars (config-only)."""
|
||||
gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py"
|
||||
# See note in test_gateway_has_auxiliary_bridge — pin UTF-8 so the
|
||||
# test runs on Windows where the default locale is cp1252.
|
||||
content = gateway_path.read_text(encoding="utf-8")
|
||||
assert "CONTEXT_COMPRESSION_PROVIDER" not in content
|
||||
assert "CONTEXT_COMPRESSION_MODEL" not in content
|
||||
|
||||
|
||||
# ── Vision model override tests ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestVisionModelOverride:
|
||||
"""Test that AUXILIARY_VISION_MODEL env var overrides the default model in the handler."""
|
||||
|
||||
def test_env_var_overrides_default(self, monkeypatch):
|
||||
monkeypatch.setenv("AUXILIARY_VISION_MODEL", "openai/gpt-4o")
|
||||
from tools.vision_tools import _handle_vision_analyze
|
||||
with patch("tools.vision_tools.vision_analyze_tool", new_callable=MagicMock) as mock_tool:
|
||||
mock_tool.return_value = '{"success": true}'
|
||||
_handle_vision_analyze({"image_url": "http://test.jpg", "question": "test"})
|
||||
call_args = mock_tool.call_args
|
||||
# 3rd positional arg = model
|
||||
assert call_args[0][2] == "openai/gpt-4o"
|
||||
|
||||
def test_default_model_when_no_override(self, monkeypatch):
|
||||
monkeypatch.delenv("AUXILIARY_VISION_MODEL", raising=False)
|
||||
from tools.vision_tools import _handle_vision_analyze
|
||||
with patch("tools.vision_tools.vision_analyze_tool", new_callable=MagicMock) as mock_tool:
|
||||
mock_tool.return_value = '{"success": true}'
|
||||
_handle_vision_analyze({"image_url": "http://test.jpg", "question": "test"})
|
||||
call_args = mock_tool.call_args
|
||||
# With no AUXILIARY_VISION_MODEL env var, model should be None
|
||||
# (the centralized call_llm router picks the provider default)
|
||||
assert call_args[0][2] is None
|
||||
|
||||
|
||||
# ── DEFAULT_CONFIG shape tests ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDefaultConfigShape:
|
||||
"""Verify the DEFAULT_CONFIG in hermes_cli/config.py has correct auxiliary structure."""
|
||||
|
||||
def test_auxiliary_section_exists(self):
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
assert "auxiliary" in DEFAULT_CONFIG
|
||||
|
||||
def test_vision_task_structure(self):
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
vision = DEFAULT_CONFIG["auxiliary"]["vision"]
|
||||
assert "provider" in vision
|
||||
assert "model" in vision
|
||||
assert vision["provider"] == "auto"
|
||||
assert vision["model"] == ""
|
||||
|
||||
def test_web_extract_task_structure(self):
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
web = DEFAULT_CONFIG["auxiliary"]["web_extract"]
|
||||
assert "provider" in web
|
||||
assert "model" in web
|
||||
assert web["provider"] == "auto"
|
||||
assert web["model"] == ""
|
||||
|
||||
|
||||
# ── CLI defaults parity ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCLIDefaultsHaveAuxiliaryKeys:
|
||||
"""Verify cli.py load_cli_config() defaults dict does NOT include auxiliary
|
||||
(it comes from config.yaml deep merge, not hardcoded defaults)."""
|
||||
|
||||
def test_cli_defaults_can_merge_auxiliary(self):
|
||||
"""The load_cli_config deep merge logic handles keys not in defaults.
|
||||
Verify auxiliary would be picked up from config.yaml."""
|
||||
# This is a structural assertion: cli.py's second-pass loop
|
||||
# carries over keys from file_config that aren't in defaults.
|
||||
# So auxiliary config from config.yaml gets merged even though
|
||||
# cli.py's defaults dict doesn't define it.
|
||||
import cli as _cli_mod
|
||||
# See note in test_gateway_has_auxiliary_bridge — pin UTF-8 so the
|
||||
# test runs on Windows where the default locale is cp1252.
|
||||
source = Path(_cli_mod.__file__).read_text(encoding="utf-8")
|
||||
assert "auxiliary_config = defaults.get(\"auxiliary\"" in source
|
||||
assert "AUXILIARY_VISION_PROVIDER" in source
|
||||
assert "AUXILIARY_VISION_MODEL" in source
|
||||
@@ -0,0 +1,413 @@
|
||||
"""Regression tests for the ``auto`` → main-model-first policy.
|
||||
|
||||
Prior to this change, aggregator users (OpenRouter / Nous Portal) had aux
|
||||
tasks routed through a cheap provider-side default (Gemini Flash) while
|
||||
non-aggregator users got their main model. This made behavior inconsistent
|
||||
and surprising — users picked Claude but got Gemini Flash summaries.
|
||||
|
||||
The current policy: ``auto`` means "use my main chat model" for every user,
|
||||
regardless of provider type. Explicit per-task overrides in ``config.yaml``
|
||||
(``auxiliary.<task>.provider``) still win. The cheap fallback chain only
|
||||
runs when the main provider has no working client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
|
||||
# ── Text aux tasks — _resolve_auto ──────────────────────────────────────────
|
||||
|
||||
|
||||
class TestResolveAutoMainFirst:
|
||||
"""_resolve_auto() must prefer main provider + main model for every user."""
|
||||
|
||||
def test_openrouter_main_uses_main_model_for_aux(self, monkeypatch):
|
||||
"""OpenRouter main user → aux uses their picked OR model, not Gemini Flash."""
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key")
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider",
|
||||
return_value="openrouter",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="anthropic/claude-sonnet-4.6",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client"
|
||||
) as mock_resolve:
|
||||
mock_client = MagicMock()
|
||||
mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6")
|
||||
|
||||
from agent.auxiliary_client import _resolve_auto
|
||||
|
||||
client, model = _resolve_auto()
|
||||
|
||||
assert client is mock_client
|
||||
assert model == "anthropic/claude-sonnet-4.6"
|
||||
# Verify it asked resolve_provider_client for the MAIN provider+model,
|
||||
# not a fallback-chain provider
|
||||
mock_resolve.assert_called_once()
|
||||
assert mock_resolve.call_args.args[0] == "openrouter"
|
||||
assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6"
|
||||
|
||||
def test_nous_main_uses_main_model_for_aux(self, monkeypatch):
|
||||
"""Nous Portal main user → aux uses their picked Nous model, not free-tier MiMo."""
|
||||
# No OPENROUTER_API_KEY → ensures if main failed we'd fall to chain
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="nous",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="anthropic/claude-opus-4.6",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client"
|
||||
) as mock_resolve:
|
||||
mock_client = MagicMock()
|
||||
mock_resolve.return_value = (mock_client, "anthropic/claude-opus-4.6")
|
||||
|
||||
from agent.auxiliary_client import _resolve_auto
|
||||
|
||||
client, model = _resolve_auto()
|
||||
|
||||
assert client is mock_client
|
||||
assert model == "anthropic/claude-opus-4.6"
|
||||
assert mock_resolve.call_args.args[0] == "nous"
|
||||
|
||||
def test_non_aggregator_main_still_uses_main(self, monkeypatch):
|
||||
"""Non-aggregator main (DeepSeek) → unchanged behavior, main model used."""
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "ds-test")
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="deepseek",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model", return_value="deepseek-chat",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client"
|
||||
) as mock_resolve:
|
||||
mock_client = MagicMock()
|
||||
mock_resolve.return_value = (mock_client, "deepseek-chat")
|
||||
|
||||
from agent.auxiliary_client import _resolve_auto
|
||||
|
||||
client, model = _resolve_auto()
|
||||
|
||||
assert client is mock_client
|
||||
assert model == "deepseek-chat"
|
||||
assert mock_resolve.call_args.args[0] == "deepseek"
|
||||
|
||||
def test_main_unavailable_falls_through_to_chain(self, monkeypatch):
|
||||
"""Main provider with no working client → fall back to aux chain."""
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
|
||||
chain_client = MagicMock()
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="anthropic",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model", return_value="claude-opus",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client",
|
||||
return_value=(None, None), # main provider has no client
|
||||
), patch(
|
||||
"agent.auxiliary_client._try_openrouter",
|
||||
return_value=(chain_client, "google/gemini-3-flash-preview"),
|
||||
):
|
||||
from agent.auxiliary_client import _resolve_auto
|
||||
|
||||
client, model = _resolve_auto()
|
||||
|
||||
assert client is chain_client
|
||||
assert model == "google/gemini-3-flash-preview"
|
||||
|
||||
def test_no_main_config_uses_chain_directly(self):
|
||||
"""No main provider configured → skip step 1, use chain (no regression)."""
|
||||
chain_client = MagicMock()
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model", return_value="",
|
||||
), patch(
|
||||
"agent.auxiliary_client._try_openrouter",
|
||||
return_value=(chain_client, "google/gemini-3-flash-preview"),
|
||||
):
|
||||
from agent.auxiliary_client import _resolve_auto
|
||||
|
||||
client, model = _resolve_auto()
|
||||
|
||||
assert client is chain_client
|
||||
|
||||
def test_runtime_override_wins_over_config(self, monkeypatch):
|
||||
"""main_runtime kwarg overrides config-read main provider/model."""
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider",
|
||||
return_value="openrouter",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model", return_value="config-model",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client"
|
||||
) as mock_resolve:
|
||||
mock_resolve.return_value = (MagicMock(), "runtime-model")
|
||||
|
||||
from agent.auxiliary_client import _resolve_auto
|
||||
|
||||
_resolve_auto(main_runtime={
|
||||
"provider": "anthropic",
|
||||
"model": "runtime-model",
|
||||
"base_url": "",
|
||||
"api_key": "",
|
||||
"api_mode": "",
|
||||
})
|
||||
|
||||
# Runtime override wins
|
||||
assert mock_resolve.call_args.args[0] == "anthropic"
|
||||
assert mock_resolve.call_args.args[1] == "runtime-model"
|
||||
|
||||
|
||||
# ── Vision — resolve_vision_provider_client ─────────────────────────────────
|
||||
|
||||
|
||||
class TestResolveVisionMainFirst:
|
||||
"""Vision auto-detection prefers the main provider first."""
|
||||
|
||||
def test_openrouter_main_vision_uses_main_model(self, monkeypatch):
|
||||
"""OpenRouter main with vision-capable model → aux vision uses main model."""
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="openrouter",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="anthropic/claude-sonnet-4.6",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client"
|
||||
) as mock_resolve, patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
):
|
||||
mock_client = MagicMock()
|
||||
mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6")
|
||||
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert provider == "openrouter"
|
||||
assert client is mock_client
|
||||
assert model == "anthropic/claude-sonnet-4.6"
|
||||
# Verify it did NOT call the strict vision backend for OpenRouter
|
||||
# (which would have used a cheap gemini-flash-preview default)
|
||||
mock_resolve.assert_called_once()
|
||||
assert mock_resolve.call_args.args[0] == "openrouter"
|
||||
assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6"
|
||||
assert mock_resolve.call_args.kwargs.get("is_vision") is True
|
||||
|
||||
def test_nous_main_vision_uses_paid_nous_vision_backend(self):
|
||||
"""Paid Nous main → aux vision uses the dedicated Nous vision backend."""
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="nous",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="openai/gpt-5",
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_strict_vision_backend",
|
||||
return_value=(MagicMock(), "google/gemini-3-flash-preview"),
|
||||
):
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert provider == "nous"
|
||||
assert client is not None
|
||||
assert model == "google/gemini-3-flash-preview"
|
||||
|
||||
def test_nous_main_vision_uses_free_tier_nous_vision_backend(self):
|
||||
"""Free-tier Nous main → aux vision uses MiMo omni, not the text main model."""
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="nous",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="xiaomi/mimo-v2-pro",
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_strict_vision_backend",
|
||||
return_value=(MagicMock(), "xiaomi/mimo-v2-omni"),
|
||||
):
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert provider == "nous"
|
||||
assert client is not None
|
||||
assert model == "xiaomi/mimo-v2-omni"
|
||||
|
||||
def test_exotic_provider_with_vision_override_preserved(self):
|
||||
"""xiaomi → mimo-v2.5 override still wins over main_model."""
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="xiaomi",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="mimo-v2-pro", # text model
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client"
|
||||
) as mock_resolve, patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
):
|
||||
mock_resolve.return_value = (MagicMock(), "mimo-v2.5")
|
||||
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert provider == "xiaomi"
|
||||
# Should use mimo-v2.5 (vision override), not mimo-v2-pro (text main)
|
||||
assert mock_resolve.call_args.args[1] == "mimo-v2.5"
|
||||
assert mock_resolve.call_args.kwargs.get("is_vision") is True
|
||||
|
||||
def test_copilot_vision_sets_vision_header(self, monkeypatch):
|
||||
"""Copilot vision requests include the header required for vision routing."""
|
||||
monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "ghu_test-token")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_headers(*, is_agent_turn=False, is_vision=False):
|
||||
captured["is_agent_turn"] = is_agent_turn
|
||||
captured["is_vision"] = is_vision
|
||||
return {"Copilot-Vision-Request": "true"} if is_vision else {}
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="copilot",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model", return_value="configured-copilot-model",
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
), patch(
|
||||
"agent.auxiliary_client.OpenAI",
|
||||
) as mock_openai, patch(
|
||||
"hermes_cli.auth.resolve_api_key_provider_credentials",
|
||||
return_value={
|
||||
"provider": "copilot",
|
||||
"api_key": "copilot-api-token",
|
||||
"base_url": "https://api.githubcopilot.com",
|
||||
},
|
||||
), patch(
|
||||
"hermes_cli.copilot_auth.copilot_request_headers",
|
||||
side_effect=fake_headers,
|
||||
):
|
||||
mock_client = MagicMock()
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert provider == "copilot"
|
||||
assert client is mock_client
|
||||
assert model == "configured-copilot-model"
|
||||
assert captured == {"is_agent_turn": True, "is_vision": True}
|
||||
assert mock_openai.call_args.kwargs["default_headers"]["Copilot-Vision-Request"] == "true"
|
||||
|
||||
def test_text_copilot_does_not_set_vision_header(self, monkeypatch):
|
||||
"""Text Copilot requests keep the vision-only header off."""
|
||||
monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "ghu_test-token")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_headers(*, is_agent_turn=False, is_vision=False):
|
||||
captured["is_agent_turn"] = is_agent_turn
|
||||
captured["is_vision"] = is_vision
|
||||
return {"Copilot-Vision-Request": "true"} if is_vision else {}
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client.OpenAI",
|
||||
) as mock_openai, patch(
|
||||
"hermes_cli.auth.resolve_api_key_provider_credentials",
|
||||
return_value={
|
||||
"provider": "copilot",
|
||||
"api_key": "copilot-api-token",
|
||||
"base_url": "https://api.githubcopilot.com",
|
||||
},
|
||||
), patch(
|
||||
"hermes_cli.copilot_auth.copilot_request_headers",
|
||||
side_effect=fake_headers,
|
||||
):
|
||||
mock_client = MagicMock()
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
|
||||
client, model = resolve_provider_client("copilot", "gpt-5-mini")
|
||||
|
||||
assert client is mock_client
|
||||
assert model == "gpt-5-mini"
|
||||
assert captured == {"is_agent_turn": True, "is_vision": False}
|
||||
assert "default_headers" not in mock_openai.call_args.kwargs
|
||||
|
||||
def test_main_unavailable_vision_falls_through_to_aggregators(self):
|
||||
"""Main provider fails → fall back to OpenRouter/Nous strict backends."""
|
||||
fallback_client = MagicMock()
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="deepseek",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model", return_value="deepseek-chat",
|
||||
), patch(
|
||||
"agent.auxiliary_client.resolve_provider_client",
|
||||
return_value=(None, None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_strict_vision_backend",
|
||||
return_value=(fallback_client, "google/gemini-3-flash-preview"),
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
):
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert client is fallback_client
|
||||
assert provider in {"openrouter", "nous"}
|
||||
|
||||
def test_explicit_provider_override_still_wins(self):
|
||||
"""Explicit config override bypasses main-first policy."""
|
||||
with patch(
|
||||
"agent.auxiliary_client._read_main_provider", return_value="openrouter",
|
||||
), patch(
|
||||
"agent.auxiliary_client._read_main_model",
|
||||
return_value="anthropic/claude-opus-4.6",
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("nous", None, None, None, None), # explicit override
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_strict_vision_backend"
|
||||
) as mock_strict:
|
||||
mock_strict.return_value = (MagicMock(), "nous-default-model")
|
||||
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
# Explicit "nous" override → uses strict backend, NOT main model path
|
||||
assert provider == "nous"
|
||||
mock_strict.assert_called_once_with("nous", None)
|
||||
|
||||
|
||||
# ── Constant cleanup ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_aggregator_providers_constant_removed():
|
||||
"""The dead _AGGREGATOR_PROVIDERS constant should no longer live in the module.
|
||||
|
||||
Removed when the main-first policy made the aggregator-skip guard obsolete.
|
||||
"""
|
||||
import agent.auxiliary_client as aux_mod
|
||||
|
||||
assert not hasattr(aux_mod, "_AGGREGATOR_PROVIDERS"), (
|
||||
"_AGGREGATOR_PROVIDERS was removed when _resolve_auto stopped "
|
||||
"treating aggregators specially. If you re-added it, the main-first "
|
||||
"policy may have regressed."
|
||||
)
|
||||
@@ -0,0 +1,493 @@
|
||||
"""Tests for named custom provider and 'main' alias resolution in auxiliary_client."""
|
||||
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate(tmp_path, monkeypatch):
|
||||
"""Redirect HERMES_HOME and clear module caches."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
# Write a minimal config so load_config doesn't fail
|
||||
(hermes_home / "config.yaml").write_text("model:\n default: test-model\n")
|
||||
|
||||
|
||||
def _write_config(tmp_path, config_dict):
|
||||
"""Write a config.yaml to the test HERMES_HOME."""
|
||||
import yaml
|
||||
config_path = tmp_path / ".hermes" / "config.yaml"
|
||||
config_path.write_text(yaml.dump(config_dict))
|
||||
|
||||
|
||||
class TestNormalizeVisionProvider:
|
||||
"""_normalize_vision_provider should resolve 'main' to actual main provider."""
|
||||
|
||||
def test_main_resolves_to_named_custom(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "my-model", "provider": "custom:beans"},
|
||||
"custom_providers": [{"name": "beans", "base_url": "http://localhost/v1"}],
|
||||
})
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("main") == "custom:beans"
|
||||
|
||||
def test_main_resolves_to_openrouter(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "anthropic/claude-sonnet-4", "provider": "openrouter"},
|
||||
})
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("main") == "openrouter"
|
||||
|
||||
def test_main_resolves_to_deepseek(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "deepseek-chat", "provider": "deepseek"},
|
||||
})
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("main") == "deepseek"
|
||||
|
||||
def test_main_falls_back_to_custom_when_no_provider(self, tmp_path):
|
||||
_write_config(tmp_path, {"model": {"default": "gpt-4o"}})
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("main") == "custom"
|
||||
|
||||
def test_bare_provider_name_unchanged(self):
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("beans") == "beans"
|
||||
assert _normalize_vision_provider("deepseek") == "deepseek"
|
||||
|
||||
def test_custom_colon_named_provider_preserved(self):
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("custom:beans") == "beans"
|
||||
|
||||
def test_codex_alias_still_works(self):
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("codex") == "openai-codex"
|
||||
|
||||
def test_auto_unchanged(self):
|
||||
from agent.auxiliary_client import _normalize_vision_provider
|
||||
assert _normalize_vision_provider("auto") == "auto"
|
||||
assert _normalize_vision_provider(None) == "auto"
|
||||
|
||||
|
||||
class TestResolveProviderClientMainAlias:
|
||||
"""resolve_provider_client('main', ...) should resolve to actual main provider."""
|
||||
|
||||
def test_main_resolves_to_named_custom_provider(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "my-model", "provider": "beans"},
|
||||
"custom_providers": [
|
||||
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, model = resolve_provider_client("main", "override-model")
|
||||
assert client is not None
|
||||
assert model == "override-model"
|
||||
assert "beans.local" in str(client.base_url)
|
||||
|
||||
def test_main_with_custom_colon_prefix(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "my-model", "provider": "custom:beans"},
|
||||
"custom_providers": [
|
||||
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, model = resolve_provider_client("main", "test")
|
||||
assert client is not None
|
||||
assert "beans.local" in str(client.base_url)
|
||||
|
||||
def test_main_resolves_github_copilot_alias(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "gpt-5.4", "provider": "github-copilot"},
|
||||
})
|
||||
with (
|
||||
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
||||
"api_key": "ghu_test_token",
|
||||
"base_url": "https://api.githubcopilot.com",
|
||||
}),
|
||||
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
||||
):
|
||||
mock_openai.return_value = MagicMock()
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
|
||||
client, model = resolve_provider_client("main", "gpt-5.4")
|
||||
|
||||
assert client is not None
|
||||
assert model == "gpt-5.4"
|
||||
assert mock_openai.called
|
||||
|
||||
|
||||
class TestResolveProviderClientNamedCustom:
|
||||
"""resolve_provider_client should resolve named custom providers directly."""
|
||||
|
||||
def test_named_custom_provider(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "test-model"},
|
||||
"custom_providers": [
|
||||
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, model = resolve_provider_client("beans", "my-model")
|
||||
assert client is not None
|
||||
assert model == "my-model"
|
||||
assert "beans.local" in str(client.base_url)
|
||||
|
||||
def test_named_custom_provider_default_model(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "main-model"},
|
||||
"custom_providers": [
|
||||
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, model = resolve_provider_client("beans")
|
||||
assert client is not None
|
||||
# Should use _read_main_model() fallback
|
||||
assert model == "main-model"
|
||||
|
||||
def test_named_custom_no_api_key_uses_fallback(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "test"},
|
||||
"custom_providers": [
|
||||
{"name": "local", "base_url": "http://localhost:8080/v1"},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, model = resolve_provider_client("local", "test")
|
||||
assert client is not None
|
||||
# no-key-required should be used
|
||||
|
||||
def test_nonexistent_named_custom_falls_through(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "test"},
|
||||
"custom_providers": [
|
||||
{"name": "beans", "base_url": "http://beans.local/v1"},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
# "coffee" doesn't exist in custom_providers
|
||||
client, model = resolve_provider_client("coffee", "test")
|
||||
assert client is None
|
||||
|
||||
|
||||
class TestResolveProviderClientModelNormalization:
|
||||
"""Direct-provider auxiliary routing should normalize models like main runtime."""
|
||||
|
||||
def test_matching_native_prefix_is_stripped_for_main_provider(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "zai/glm-5.1", "provider": "zai"},
|
||||
})
|
||||
with (
|
||||
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
||||
"api_key": "glm-key",
|
||||
"base_url": "https://api.z.ai/api/paas/v4",
|
||||
}),
|
||||
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
||||
):
|
||||
mock_openai.return_value = MagicMock()
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
|
||||
client, model = resolve_provider_client("main", "zai/glm-5.1")
|
||||
|
||||
assert client is not None
|
||||
assert model == "glm-5.1"
|
||||
|
||||
def test_non_matching_prefix_is_preserved_for_direct_provider(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "zai/glm-5.1", "provider": "zai"},
|
||||
})
|
||||
with (
|
||||
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
||||
"api_key": "glm-key",
|
||||
"base_url": "https://api.z.ai/api/paas/v4",
|
||||
}),
|
||||
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
||||
):
|
||||
mock_openai.return_value = MagicMock()
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
|
||||
client, model = resolve_provider_client("zai", "google/gemini-2.5-pro")
|
||||
|
||||
assert client is not None
|
||||
assert model == "google/gemini-2.5-pro"
|
||||
|
||||
def test_aggregator_vendor_slug_is_preserved(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
|
||||
client, model = resolve_provider_client(
|
||||
"openrouter", "anthropic/claude-sonnet-4.6"
|
||||
)
|
||||
|
||||
assert client is not None
|
||||
assert model == "anthropic/claude-sonnet-4.6"
|
||||
|
||||
|
||||
class TestResolveVisionProviderClientModelNormalization:
|
||||
"""Vision auto-routing should reuse the same provider-specific normalization."""
|
||||
|
||||
def test_vision_auto_strips_matching_main_provider_prefix(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "zai/glm-5.1", "provider": "zai"},
|
||||
})
|
||||
with (
|
||||
patch("agent.auxiliary_client._read_nous_auth", return_value=None),
|
||||
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
||||
"api_key": "glm-key",
|
||||
"base_url": "https://api.z.ai/api/paas/v4",
|
||||
}),
|
||||
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
||||
):
|
||||
mock_openai.return_value = MagicMock()
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client()
|
||||
|
||||
assert provider == "zai"
|
||||
assert client is not None
|
||||
assert model == "glm-5v-turbo" # zai has dedicated vision model in _PROVIDER_VISION_MODELS
|
||||
|
||||
|
||||
class TestVisionPathApiMode:
|
||||
"""Vision path should propagate api_mode to _get_cached_client."""
|
||||
|
||||
def test_explicit_provider_passes_api_mode(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"default": "test-model"},
|
||||
"auxiliary": {"vision": {"api_mode": "chat_completions"}},
|
||||
})
|
||||
with patch("agent.auxiliary_client._get_cached_client") as mock_gcc:
|
||||
mock_gcc.return_value = (MagicMock(), "test-model")
|
||||
from agent.auxiliary_client import resolve_vision_provider_client
|
||||
|
||||
provider, client, model = resolve_vision_provider_client(provider="deepseek")
|
||||
|
||||
mock_gcc.assert_called_once()
|
||||
_, kwargs = mock_gcc.call_args
|
||||
assert kwargs.get("api_mode") == "chat_completions"
|
||||
|
||||
|
||||
class TestProvidersDictApiModeAnthropicMessages:
|
||||
"""Regression guard for #15033.
|
||||
|
||||
Named providers declared under the ``providers:`` dict with
|
||||
``api_mode: anthropic_messages`` must route auxiliary calls through
|
||||
the Anthropic Messages API (via AnthropicAuxiliaryClient), not
|
||||
through an OpenAI chat-completions client.
|
||||
|
||||
The bug had two halves: the providers-dict branch of
|
||||
``_get_named_custom_provider`` dropped the ``api_mode`` field, and
|
||||
``resolve_provider_client``'s named-custom branch never read it.
|
||||
"""
|
||||
|
||||
def test_providers_dict_propagates_api_mode(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
|
||||
_write_config(tmp_path, {
|
||||
"providers": {
|
||||
"myrelay": {
|
||||
"name": "myrelay",
|
||||
"base_url": "https://example-relay.test/anthropic",
|
||||
"key_env": "MYRELAY_API_KEY",
|
||||
"api_mode": "anthropic_messages",
|
||||
"default_model": "claude-opus-4-7",
|
||||
},
|
||||
},
|
||||
})
|
||||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||||
entry = _get_named_custom_provider("myrelay")
|
||||
assert entry is not None
|
||||
assert entry.get("api_mode") == "anthropic_messages"
|
||||
assert entry.get("base_url") == "https://example-relay.test/anthropic"
|
||||
assert entry.get("api_key") == "sk-test"
|
||||
|
||||
def test_providers_dict_invalid_api_mode_is_dropped(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"providers": {
|
||||
"weird": {
|
||||
"name": "weird",
|
||||
"base_url": "https://example.test",
|
||||
"api_mode": "bogus_nonsense",
|
||||
"default_model": "x",
|
||||
},
|
||||
},
|
||||
})
|
||||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||||
entry = _get_named_custom_provider("weird")
|
||||
assert entry is not None
|
||||
assert "api_mode" not in entry
|
||||
|
||||
def test_providers_dict_without_api_mode_is_unchanged(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"providers": {
|
||||
"localchat": {
|
||||
"name": "localchat",
|
||||
"base_url": "http://127.0.0.1:1234/v1",
|
||||
"api_key": "local-key",
|
||||
"default_model": "llama-3",
|
||||
},
|
||||
},
|
||||
})
|
||||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||||
entry = _get_named_custom_provider("localchat")
|
||||
assert entry is not None
|
||||
assert "api_mode" not in entry
|
||||
|
||||
def test_resolve_provider_client_returns_anthropic_client(self, tmp_path, monkeypatch):
|
||||
"""Named custom provider with api_mode=anthropic_messages must
|
||||
route through AnthropicAuxiliaryClient."""
|
||||
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
|
||||
_write_config(tmp_path, {
|
||||
"providers": {
|
||||
"myrelay": {
|
||||
"name": "myrelay",
|
||||
"base_url": "https://example-relay.test/anthropic",
|
||||
"key_env": "MYRELAY_API_KEY",
|
||||
"api_mode": "anthropic_messages",
|
||||
"default_model": "claude-opus-4-7",
|
||||
},
|
||||
},
|
||||
})
|
||||
from agent.auxiliary_client import (
|
||||
resolve_provider_client,
|
||||
AnthropicAuxiliaryClient,
|
||||
AsyncAnthropicAuxiliaryClient,
|
||||
)
|
||||
sync_client, sync_model = resolve_provider_client("myrelay", async_mode=False)
|
||||
assert isinstance(sync_client, AnthropicAuxiliaryClient), (
|
||||
f"expected AnthropicAuxiliaryClient, got {type(sync_client).__name__}"
|
||||
)
|
||||
assert sync_model == "claude-opus-4-7"
|
||||
|
||||
async_client, async_model = resolve_provider_client("myrelay", async_mode=True)
|
||||
assert isinstance(async_client, AsyncAnthropicAuxiliaryClient), (
|
||||
f"expected AsyncAnthropicAuxiliaryClient, got {type(async_client).__name__}"
|
||||
)
|
||||
assert async_model == "claude-opus-4-7"
|
||||
|
||||
def test_aux_task_override_routes_named_provider_to_anthropic(self, tmp_path, monkeypatch):
|
||||
"""The full chain: auxiliary.<task>.provider: myrelay with
|
||||
api_mode anthropic_messages must produce an Anthropic client."""
|
||||
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
|
||||
_write_config(tmp_path, {
|
||||
"providers": {
|
||||
"myrelay": {
|
||||
"name": "myrelay",
|
||||
"base_url": "https://example-relay.test/anthropic",
|
||||
"key_env": "MYRELAY_API_KEY",
|
||||
"api_mode": "anthropic_messages",
|
||||
"default_model": "claude-opus-4-7",
|
||||
},
|
||||
},
|
||||
"auxiliary": {
|
||||
"compression": {
|
||||
"provider": "myrelay",
|
||||
"model": "claude-sonnet-4.6",
|
||||
},
|
||||
},
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
|
||||
})
|
||||
from agent.auxiliary_client import (
|
||||
get_async_text_auxiliary_client,
|
||||
get_text_auxiliary_client,
|
||||
AnthropicAuxiliaryClient,
|
||||
AsyncAnthropicAuxiliaryClient,
|
||||
)
|
||||
async_client, async_model = get_async_text_auxiliary_client("compression")
|
||||
assert isinstance(async_client, AsyncAnthropicAuxiliaryClient)
|
||||
assert async_model == "claude-sonnet-4.6"
|
||||
|
||||
sync_client, sync_model = get_text_auxiliary_client("compression")
|
||||
assert isinstance(sync_client, AnthropicAuxiliaryClient)
|
||||
assert sync_model == "claude-sonnet-4.6"
|
||||
|
||||
def test_provider_without_api_mode_still_uses_openai(self, tmp_path):
|
||||
"""Named providers that don't declare api_mode should still go
|
||||
through the plain OpenAI-wire path (no regression)."""
|
||||
_write_config(tmp_path, {
|
||||
"providers": {
|
||||
"localchat": {
|
||||
"name": "localchat",
|
||||
"base_url": "http://127.0.0.1:1234/v1",
|
||||
"api_key": "local-key",
|
||||
"default_model": "llama-3",
|
||||
},
|
||||
},
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
from openai import OpenAI, AsyncOpenAI
|
||||
sync_client, _ = resolve_provider_client("localchat", async_mode=False)
|
||||
# sync returns the raw OpenAI client
|
||||
assert isinstance(sync_client, OpenAI)
|
||||
async_client, _ = resolve_provider_client("localchat", async_mode=True)
|
||||
assert isinstance(async_client, AsyncOpenAI)
|
||||
|
||||
|
||||
class TestCustomProviderAliasCollision:
|
||||
"""A user-declared custom_providers entry whose name matches a built-in
|
||||
*alias* (not a canonical provider) must win over the built-in.
|
||||
|
||||
Regression guard for #15743: users who defined fallback_model pointing at
|
||||
a custom_providers entry named ``kimi`` were having requests routed to
|
||||
the built-in kimi-coding endpoint because ``_normalize_aux_provider``
|
||||
rewrote ``kimi`` → ``kimi-coding`` before the named-custom lookup.
|
||||
"""
|
||||
|
||||
def test_custom_named_kimi_wins_over_builtin_alias(self, tmp_path):
|
||||
_write_config(tmp_path, {
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
|
||||
"custom_providers": [
|
||||
{
|
||||
"name": "kimi",
|
||||
"base_url": "https://my-custom-kimi.example.com/v1",
|
||||
"api_key": "my-kimi-key",
|
||||
"models": {"my-kimi-model": {"context_length": 200000}},
|
||||
},
|
||||
],
|
||||
})
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
from openai import OpenAI
|
||||
client, model = resolve_provider_client("kimi", model="my-kimi-model", raw_codex=True)
|
||||
assert isinstance(client, OpenAI)
|
||||
assert "my-custom-kimi.example.com" in str(client.base_url)
|
||||
assert client.api_key == "my-kimi-key"
|
||||
assert model == "my-kimi-model"
|
||||
|
||||
def test_bare_kimi_without_custom_still_routes_to_builtin(self, tmp_path, monkeypatch):
|
||||
"""Regression guard: bare 'kimi' with no custom entry must still
|
||||
reach the built-in kimi-coding provider."""
|
||||
_write_config(tmp_path, {
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
|
||||
})
|
||||
monkeypatch.setenv("KIMI_API_KEY", "builtin-kimi-key")
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, _ = resolve_provider_client("kimi", model="kimi-k2-0905-preview", raw_codex=True)
|
||||
assert client is not None
|
||||
base_url = str(client.base_url)
|
||||
# Built-in kimi-coding points at api.moonshot.ai
|
||||
assert "moonshot" in base_url or "kimi" in base_url, f"unexpected base_url {base_url!r}"
|
||||
|
||||
def test_explicit_overrides_applied_on_api_key_branch(self, tmp_path, monkeypatch):
|
||||
"""Explicit base_url/api_key from the caller must override the
|
||||
registered provider's defaults on the API-key branch. Used by
|
||||
_try_activate_fallback to route a fallback through a built-in
|
||||
provider name but targeting a user-supplied endpoint."""
|
||||
_write_config(tmp_path, {
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
|
||||
})
|
||||
monkeypatch.setenv("KIMI_API_KEY", "builtin-kimi-key")
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
from openai import OpenAI
|
||||
client, _ = resolve_provider_client(
|
||||
"kimi-coding", model="kimi-k2", raw_codex=True,
|
||||
explicit_base_url="https://override.example.com",
|
||||
explicit_api_key="override-key",
|
||||
)
|
||||
assert isinstance(client, OpenAI)
|
||||
assert "override.example.com" in str(client.base_url)
|
||||
assert client.api_key == "override-key"
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Tests for transport auto-detection in agent.auxiliary_client.
|
||||
|
||||
Auxiliary clients must pick the correct wire protocol (OpenAI
|
||||
chat.completions vs native Anthropic Messages) based on the endpoint,
|
||||
regardless of which resolve_provider_client branch built them.
|
||||
|
||||
Regression target (April 2026): Kimi Coding Plan's ``api.kimi.com/coding``
|
||||
endpoint only speaks Anthropic Messages — sending ``kimi-for-coding`` over
|
||||
chat.completions returns 404 "resource_not_found_error". The named
|
||||
``kimi-coding`` provider branch in resolve_provider_client used to build a
|
||||
plain OpenAI client, so title generation / vision / compression /
|
||||
web_extract all failed on Kimi Coding Plan users.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch):
|
||||
for key in (
|
||||
"OPENAI_API_KEY", "OPENAI_BASE_URL",
|
||||
"ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN",
|
||||
"KIMI_API_KEY", "KIMI_CODING_API_KEY", "KIMI_BASE_URL",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# URL detection helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("url,expected,label", [
|
||||
("https://api.kimi.com/coding/v1", True, "Kimi Coding Plan /v1"),
|
||||
("https://api.kimi.com/coding", True, "Kimi Coding Plan no /v1"),
|
||||
("https://api.moonshot.ai/v1", False, "Moonshot legacy"),
|
||||
("https://api.minimax.io/anthropic", True, "MiniMax /anthropic"),
|
||||
("https://litellm.example.com/v1/anthropic", True, "/anthropic suffix"),
|
||||
("https://api.anthropic.com", True, "native Anthropic"),
|
||||
("https://api.anthropic.com/v1", True, "native Anthropic /v1"),
|
||||
("https://openrouter.ai/api/v1", False, "OpenRouter"),
|
||||
("https://api.openai.com/v1", False, "OpenAI"),
|
||||
("https://inference-api.nousresearch.com/v1", False, "Nous"),
|
||||
("", False, "empty"),
|
||||
(None, False, "None"),
|
||||
])
|
||||
def test_endpoint_speaks_anthropic_messages(url, expected, label):
|
||||
from agent.auxiliary_client import _endpoint_speaks_anthropic_messages
|
||||
assert _endpoint_speaks_anthropic_messages(url) is expected, (
|
||||
f"{label}: {url!r} should be {expected}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _maybe_wrap_anthropic decision table
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_maybe_wrap_anthropic_rewraps_kimi_coding_url():
|
||||
"""Plain OpenAI client pointed at api.kimi.com/coding gets rewrapped."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
plain_client = MagicMock(name="plain_openai")
|
||||
fake_anthropic = MagicMock(name="anthropic_sdk_client")
|
||||
|
||||
with patch(
|
||||
"agent.anthropic_adapter.build_anthropic_client",
|
||||
return_value=fake_anthropic,
|
||||
):
|
||||
result = _maybe_wrap_anthropic(
|
||||
plain_client, "kimi-for-coding", "sk-kimi-test",
|
||||
"https://api.kimi.com/coding", api_mode=None,
|
||||
)
|
||||
assert isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_rewraps_slash_anthropic_url():
|
||||
"""Plain OpenAI client pointed at any /anthropic URL gets rewrapped."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
plain_client = MagicMock(name="plain_openai")
|
||||
fake_anthropic = MagicMock(name="anthropic_sdk_client")
|
||||
|
||||
with patch(
|
||||
"agent.anthropic_adapter.build_anthropic_client",
|
||||
return_value=fake_anthropic,
|
||||
):
|
||||
result = _maybe_wrap_anthropic(
|
||||
plain_client, "MiniMax-M2.7", "mm-key",
|
||||
"https://api.minimax.io/anthropic", api_mode=None,
|
||||
)
|
||||
assert isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_skips_openai_wire_urls():
|
||||
"""OpenRouter / OpenAI / Moonshot-legacy stay as plain OpenAI clients."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
plain_client = MagicMock(name="plain_openai")
|
||||
# No patch on build_anthropic_client — if the function tried to call it,
|
||||
# we'd get an AttributeError-style failure. The point is it shouldn't.
|
||||
result = _maybe_wrap_anthropic(
|
||||
plain_client, "claude-sonnet-4.6", "sk-or-test",
|
||||
"https://openrouter.ai/api/v1", api_mode=None,
|
||||
)
|
||||
assert result is plain_client
|
||||
assert not isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_respects_explicit_chat_completions():
|
||||
"""api_mode=chat_completions overrides URL heuristics."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
plain_client = MagicMock(name="plain_openai")
|
||||
result = _maybe_wrap_anthropic(
|
||||
plain_client, "kimi-for-coding", "sk-kimi-test",
|
||||
"https://api.kimi.com/coding",
|
||||
api_mode="chat_completions", # explicit override
|
||||
)
|
||||
assert result is plain_client, "Explicit chat_completions must bypass wrap"
|
||||
assert not isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_honors_explicit_anthropic_messages():
|
||||
"""api_mode=anthropic_messages wraps even when URL wouldn't trigger."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
plain_client = MagicMock(name="plain_openai")
|
||||
fake_anthropic = MagicMock(name="anthropic_sdk_client")
|
||||
|
||||
with patch(
|
||||
"agent.anthropic_adapter.build_anthropic_client",
|
||||
return_value=fake_anthropic,
|
||||
):
|
||||
result = _maybe_wrap_anthropic(
|
||||
plain_client, "model-name", "some-key",
|
||||
"https://opaque.internal/v1", # URL alone wouldn't trigger
|
||||
api_mode="anthropic_messages",
|
||||
)
|
||||
assert isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_double_wrap_safe():
|
||||
"""Already-wrapped AnthropicAuxiliaryClient passes through unchanged."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
already_wrapped = MagicMock(spec=AnthropicAuxiliaryClient)
|
||||
result = _maybe_wrap_anthropic(
|
||||
already_wrapped, "model", "key",
|
||||
"https://api.kimi.com/coding", api_mode=None,
|
||||
)
|
||||
assert result is already_wrapped
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_codex_client_passes_through():
|
||||
"""CodexAuxiliaryClient is never re-dispatched."""
|
||||
from agent.auxiliary_client import (
|
||||
_maybe_wrap_anthropic,
|
||||
CodexAuxiliaryClient,
|
||||
AnthropicAuxiliaryClient,
|
||||
)
|
||||
|
||||
codex_client = MagicMock(spec=CodexAuxiliaryClient)
|
||||
result = _maybe_wrap_anthropic(
|
||||
codex_client, "model", "key",
|
||||
"https://api.kimi.com/coding", api_mode=None,
|
||||
)
|
||||
assert result is codex_client
|
||||
assert not isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
def test_maybe_wrap_anthropic_sdk_missing_falls_back():
|
||||
"""ImportError on anthropic SDK returns plain client with warning."""
|
||||
from agent.auxiliary_client import _maybe_wrap_anthropic, AnthropicAuxiliaryClient
|
||||
|
||||
plain_client = MagicMock(name="plain_openai")
|
||||
|
||||
def _raise_import(*args, **kwargs):
|
||||
raise ImportError("no anthropic SDK")
|
||||
|
||||
with patch(
|
||||
"agent.anthropic_adapter.build_anthropic_client",
|
||||
side_effect=_raise_import,
|
||||
):
|
||||
# The ImportError is caught on the `from ... import` line inside
|
||||
# _maybe_wrap_anthropic, which runs before build_anthropic_client is
|
||||
# called. To exercise the ImportError path we need to patch the
|
||||
# module lookup itself.
|
||||
import sys as _sys
|
||||
saved = _sys.modules.get("agent.anthropic_adapter")
|
||||
_sys.modules["agent.anthropic_adapter"] = None # force ImportError
|
||||
try:
|
||||
result = _maybe_wrap_anthropic(
|
||||
plain_client, "kimi-for-coding", "sk-kimi-test",
|
||||
"https://api.kimi.com/coding", api_mode=None,
|
||||
)
|
||||
finally:
|
||||
if saved is not None:
|
||||
_sys.modules["agent.anthropic_adapter"] = saved
|
||||
else:
|
||||
_sys.modules.pop("agent.anthropic_adapter", None)
|
||||
|
||||
assert result is plain_client
|
||||
assert not isinstance(result, AnthropicAuxiliaryClient)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration: resolve_provider_client for named kimi-coding provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_resolve_provider_client_kimi_coding_wraps_anthropic(monkeypatch, tmp_path):
|
||||
"""End-to-end: resolve_provider_client('kimi-coding', 'kimi-for-coding')
|
||||
must return AnthropicAuxiliaryClient because /coding speaks Anthropic.
|
||||
|
||||
This is the primary regression guard: the bug that caused title
|
||||
generation 404s on every Kimi Coding Plan user after the "main model
|
||||
for every user" aux design shipped.
|
||||
"""
|
||||
from agent.auxiliary_client import (
|
||||
resolve_provider_client,
|
||||
AnthropicAuxiliaryClient,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
# sk-kimi- prefix triggers /coding endpoint auto-detection
|
||||
monkeypatch.setenv("KIMI_API_KEY", "sk-kimi-faketesttoken123")
|
||||
|
||||
client, model = resolve_provider_client("kimi-coding", "kimi-for-coding")
|
||||
assert client is not None, "Should resolve a client"
|
||||
assert isinstance(client, AnthropicAuxiliaryClient), (
|
||||
"Kimi Coding Plan endpoint (api.kimi.com/coding) speaks Anthropic "
|
||||
"Messages — aux client MUST be AnthropicAuxiliaryClient, got "
|
||||
f"{type(client).__name__}"
|
||||
)
|
||||
assert "kimi.com/coding" in str(client.base_url)
|
||||
@@ -0,0 +1,661 @@
|
||||
"""Tests for the Microsoft Entra ID adapter (agent/azure_identity_adapter.py).
|
||||
|
||||
Covers:
|
||||
- Scope resolution per Azure host shape
|
||||
- Display masking for callable + string + None inputs
|
||||
- Cache-fingerprint stability under callable refresh
|
||||
- is_token_provider truthiness on callables vs strings
|
||||
- EntraIdentityConfig serialization round-trip
|
||||
- Token provider construction with mocked azure-identity
|
||||
- Credential cache reuse + reset
|
||||
- has_azure_identity_credentials timeout / failure paths
|
||||
- describe_active_credential structural reporting
|
||||
- Lazy-install error path when azure-identity absent + lazy installs
|
||||
disabled
|
||||
|
||||
We mock azure.identity at the import boundary rather than hitting any
|
||||
real Azure endpoint. Tests must remain hermetic per AGENTS.md.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure we always import a fresh adapter module — credential caches in
|
||||
# the adapter persist across tests otherwise, polluting assertions
|
||||
# about cache invalidation.
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_adapter_cache():
|
||||
from agent.azure_identity_adapter import reset_credential_cache
|
||||
reset_credential_cache()
|
||||
yield
|
||||
reset_credential_cache()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scope constant
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEntraScopeConstant:
|
||||
"""Pin the Microsoft-documented Foundry inference scope.
|
||||
|
||||
Microsoft's official samples for both ``*.openai.azure.com`` and
|
||||
``*.services.ai.azure.com`` use ``https://ai.azure.com/.default``.
|
||||
The older ``cognitiveservices.azure.com/.default`` is the
|
||||
control-plane scope and is rejected for inference by newer
|
||||
Azure OpenAI / Foundry resources.
|
||||
|
||||
Users with sovereign-cloud or unusual-tenant requirements pass the
|
||||
scope explicitly via ``model.entra.scope`` in ``config.yaml``.
|
||||
|
||||
Refs:
|
||||
* https://learn.microsoft.com/azure/ai-foundry/openai/how-to/managed-identity
|
||||
* https://learn.microsoft.com/azure/ai-foundry/foundry-models/how-to/configure-entra-id
|
||||
"""
|
||||
|
||||
def test_default_scope_matches_microsoft_documentation(self):
|
||||
from agent.azure_identity_adapter import SCOPE_AI_AZURE_DEFAULT
|
||||
assert SCOPE_AI_AZURE_DEFAULT == "https://ai.azure.com/.default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cache fingerprint + http-bearer helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMaterializeBearerForHttp:
|
||||
"""The only helper that mints a real bearer JWT — must call the
|
||||
callable exactly once and never fall through to display masking."""
|
||||
|
||||
def test_callable_is_invoked_and_returns_token(self):
|
||||
from agent.azure_identity_adapter import materialize_bearer_for_http
|
||||
|
||||
invoked = {"count": 0}
|
||||
|
||||
def provider():
|
||||
invoked["count"] += 1
|
||||
return "fresh-jwt"
|
||||
|
||||
assert materialize_bearer_for_http(provider) == "fresh-jwt"
|
||||
assert invoked["count"] == 1
|
||||
|
||||
def test_string_passes_through(self):
|
||||
from agent.azure_identity_adapter import materialize_bearer_for_http
|
||||
assert materialize_bearer_for_http("plain-key") == "plain-key"
|
||||
|
||||
def test_callable_returning_empty_raises(self):
|
||||
from agent.azure_identity_adapter import materialize_bearer_for_http
|
||||
with pytest.raises(ValueError):
|
||||
materialize_bearer_for_http(lambda: "")
|
||||
|
||||
def test_empty_string_raises(self):
|
||||
from agent.azure_identity_adapter import materialize_bearer_for_http
|
||||
with pytest.raises(ValueError):
|
||||
materialize_bearer_for_http("")
|
||||
with pytest.raises(ValueError):
|
||||
materialize_bearer_for_http(None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_bearer_http_client — the Anthropic-on-Foundry bridge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildBearerHttpClient:
|
||||
"""``build_bearer_http_client`` returns an ``httpx.Client`` whose
|
||||
request event hook mints a fresh JWT per outbound request. This is
|
||||
how Entra ID auth reaches the Anthropic SDK (which does not accept
|
||||
callable ``auth_token``)."""
|
||||
|
||||
def test_returns_httpx_client_with_request_hook(self):
|
||||
import httpx
|
||||
from agent.azure_identity_adapter import build_bearer_http_client
|
||||
|
||||
client = build_bearer_http_client(lambda: "jwt")
|
||||
try:
|
||||
assert isinstance(client, httpx.Client)
|
||||
hooks = client.event_hooks.get("request", [])
|
||||
assert len(hooks) >= 1
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def test_hook_overrides_authorization_header(self):
|
||||
import httpx
|
||||
from agent.azure_identity_adapter import build_bearer_http_client
|
||||
|
||||
minted_tokens = []
|
||||
|
||||
def provider():
|
||||
minted_tokens.append(f"jwt-{len(minted_tokens) + 1}")
|
||||
return minted_tokens[-1]
|
||||
|
||||
client = build_bearer_http_client(provider)
|
||||
try:
|
||||
hook = client.event_hooks["request"][0]
|
||||
# Build a request with conflicting pre-set headers and verify
|
||||
# the hook strips them and installs the fresh bearer.
|
||||
req = httpx.Request(
|
||||
"POST", "https://example.com/v1/messages",
|
||||
headers={
|
||||
"Authorization": "Bearer stale-token",
|
||||
"api-key": "static-key",
|
||||
"x-api-key": "static-key",
|
||||
},
|
||||
json={"hello": "world"},
|
||||
)
|
||||
hook(req)
|
||||
assert req.headers["Authorization"] == "Bearer jwt-1"
|
||||
# The static-key headers must be stripped — sending both
|
||||
# auth values would be ambiguous on Azure.
|
||||
assert "api-key" not in req.headers
|
||||
assert "x-api-key" not in req.headers
|
||||
|
||||
# Second invocation mints a fresh token.
|
||||
req2 = httpx.Request("GET", "https://example.com/v1/models")
|
||||
hook(req2)
|
||||
assert req2.headers["Authorization"] == "Bearer jwt-2"
|
||||
assert len(minted_tokens) == 2
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def test_hook_strips_auth_headers_and_warns_when_token_provider_fails(self, caplog):
|
||||
"""When the token provider fails (chain exhausted, IMDS down, az
|
||||
login expired), the hook must:
|
||||
1. Log at WARNING level so the misconfiguration is visible at
|
||||
default log level (not buried at DEBUG).
|
||||
2. Strip any pre-set Authorization headers — including the
|
||||
placeholder ``entra-id-bearer-via-http-hook`` sentinel that
|
||||
:func:`_build_anthropic_client_with_bearer_hook` sets on the
|
||||
Anthropic SDK constructor. This produces a clean
|
||||
"missing auth" 401 from Azure rather than a sentinel-bearing
|
||||
401 that's harder to diagnose AND avoids leaking the
|
||||
sentinel string into upstream access logs.
|
||||
"""
|
||||
import logging
|
||||
import httpx
|
||||
from agent.azure_identity_adapter import build_bearer_http_client
|
||||
|
||||
def bad_provider():
|
||||
return "" # empty token → materialize_bearer_for_http raises
|
||||
|
||||
client = build_bearer_http_client(bad_provider)
|
||||
try:
|
||||
hook = client.event_hooks["request"][0]
|
||||
req = httpx.Request(
|
||||
"POST", "https://example.com/v1/messages",
|
||||
headers={
|
||||
"Authorization": "Bearer entra-id-bearer-via-http-hook",
|
||||
"api-key": "leaked-placeholder",
|
||||
},
|
||||
)
|
||||
with caplog.at_level(logging.WARNING, logger="agent.azure_identity_adapter"):
|
||||
hook(req) # Must not raise.
|
||||
# Pre-set auth headers stripped — no sentinel makes it to Azure.
|
||||
assert "Authorization" not in req.headers
|
||||
assert "api-key" not in req.headers
|
||||
# WARNING was logged so the user sees the misconfiguration.
|
||||
assert any(
|
||||
rec.levelno == logging.WARNING and "Entra ID token provider" in rec.message
|
||||
for rec in caplog.records
|
||||
)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def test_rejects_non_callable_provider(self):
|
||||
from agent.azure_identity_adapter import build_bearer_http_client
|
||||
with pytest.raises(ValueError):
|
||||
build_bearer_http_client(cast(Callable[[], str], "plain-string-not-callable"))
|
||||
with pytest.raises(ValueError):
|
||||
build_bearer_http_client(cast(Callable[[], str], None))
|
||||
|
||||
def test_forwards_httpx_kwargs(self):
|
||||
import httpx
|
||||
from agent.azure_identity_adapter import build_bearer_http_client
|
||||
|
||||
timeout = httpx.Timeout(60.0, connect=5.0)
|
||||
client = build_bearer_http_client(lambda: "jwt", timeout=timeout)
|
||||
try:
|
||||
# httpx stores the timeout per-pool; just sanity-check it was
|
||||
# accepted without TypeError.
|
||||
assert client is not None
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
class TestIsTokenProvider:
|
||||
def test_callable_is_token_provider(self):
|
||||
from agent.azure_identity_adapter import is_token_provider
|
||||
assert is_token_provider(lambda: "x") is True
|
||||
|
||||
def test_string_is_not_token_provider(self):
|
||||
from agent.azure_identity_adapter import is_token_provider
|
||||
assert is_token_provider("static-key") is False
|
||||
# ``str`` instances are technically callable in some edge cases
|
||||
# — confirm they're never classified as token providers.
|
||||
assert is_token_provider("") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EntraIdentityConfig
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEntraIdentityConfig:
|
||||
"""The serializable config that crosses multiprocessing boundaries —
|
||||
must round-trip through dict cleanly and never lose fields."""
|
||||
|
||||
def test_to_dict_round_trip(self):
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig
|
||||
cfg = EntraIdentityConfig(
|
||||
scope="https://ai.azure.com/.default",
|
||||
exclude_interactive_browser=False,
|
||||
)
|
||||
rebuilt = EntraIdentityConfig.from_dict(cfg.to_dict())
|
||||
assert rebuilt == cfg
|
||||
|
||||
def test_from_dict_handles_empty_strings(self):
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig
|
||||
cfg = EntraIdentityConfig.from_dict({
|
||||
"scope": "",
|
||||
"client_id": None,
|
||||
})
|
||||
# Empty scope falls back to default
|
||||
assert cfg.scope.endswith("/.default")
|
||||
|
||||
def test_from_dict_ignores_legacy_identity_keys(self):
|
||||
"""Old config.yaml that still has model.entra.client_id /
|
||||
tenant_id / authority should not crash from_dict — those values
|
||||
are now read from AZURE_* env vars by azure-identity directly."""
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig
|
||||
cfg = EntraIdentityConfig.from_dict({
|
||||
"tenant_id": "legacy-tenant",
|
||||
"authority": "https://login.partner.microsoftonline.cn",
|
||||
"client_id": "user-mi-client",
|
||||
})
|
||||
# Legacy keys silently ignored — no crash, no surprise field on the dataclass.
|
||||
assert not hasattr(cfg, "client_id")
|
||||
assert not hasattr(cfg, "tenant_id")
|
||||
assert not hasattr(cfg, "authority")
|
||||
|
||||
def test_constructor_normalizes_empty_scope(self):
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig
|
||||
cfg = EntraIdentityConfig(scope="")
|
||||
assert cfg.scope.endswith("/.default")
|
||||
|
||||
def test_from_dict_default_scope_override(self):
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig
|
||||
cfg = EntraIdentityConfig.from_dict(
|
||||
{"scope": ""},
|
||||
default_scope="https://custom.example/.default",
|
||||
)
|
||||
assert cfg.scope == "https://custom.example/.default"
|
||||
|
||||
def test_dataclass_is_frozen(self):
|
||||
# Frozen dataclasses are hashable / safe to pass through caches.
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig
|
||||
cfg = EntraIdentityConfig()
|
||||
with pytest.raises((AttributeError, Exception)):
|
||||
setattr(cfg, "scope", "mutated")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Credential / token provider construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeAzureIdentity:
|
||||
"""Stand-in for the ``azure.identity`` module.
|
||||
|
||||
Captures kwargs passed to ``DefaultAzureCredential`` so tests can
|
||||
assert how config flows into the SDK.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.last_credential_kwargs = None
|
||||
self.last_scope = None
|
||||
self.credential_count = 0
|
||||
|
||||
def DefaultAzureCredential(self, **kwargs): # noqa: N802 — match SDK
|
||||
self.last_credential_kwargs = kwargs
|
||||
self.credential_count += 1
|
||||
return SimpleNamespace(
|
||||
get_token=lambda scope: SimpleNamespace(token="fake-jwt", expires_on=9999999999),
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
def get_bearer_token_provider(self, credential, scope):
|
||||
self.last_scope = scope
|
||||
# Return a callable that mints a token when invoked.
|
||||
return lambda: f"jwt-for-{scope}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_azure_identity(monkeypatch):
|
||||
"""Install a fake azure.identity into sys.modules and stub the
|
||||
adapter's `_require_azure_identity` so all tests use the fake."""
|
||||
fake = _FakeAzureIdentity()
|
||||
|
||||
fake_module = SimpleNamespace(
|
||||
DefaultAzureCredential=fake.DefaultAzureCredential,
|
||||
get_bearer_token_provider=fake.get_bearer_token_provider,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "azure", SimpleNamespace(identity=fake_module))
|
||||
monkeypatch.setitem(sys.modules, "azure.identity", fake_module)
|
||||
|
||||
# The adapter's `_require_azure_identity` does its own import, so
|
||||
# patch that too to make sure tests never hit the real package's
|
||||
# singleton state.
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
monkeypatch.setattr(_adapter, "_require_azure_identity", lambda: fake_module)
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
class TestBuildCredential:
|
||||
def test_default_kwargs_are_minimal(self, fake_azure_identity):
|
||||
"""SDK default for ``exclude_interactive_browser_credential`` is
|
||||
True; we only pass it when the user opts IN to interactive
|
||||
browser auth. Tenant / authority / service principal config
|
||||
flow through the standard ``AZURE_*`` env vars (read by
|
||||
azure-identity directly), not Hermes config kwargs."""
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
|
||||
cred = build_credential(EntraIdentityConfig())
|
||||
kwargs = fake_azure_identity.last_credential_kwargs
|
||||
# Default config should produce empty kwargs — SDK uses its own
|
||||
# defaults plus env-var-driven settings.
|
||||
assert kwargs == {}
|
||||
assert cred is not None
|
||||
|
||||
def test_interactive_browser_opt_in(self, fake_azure_identity):
|
||||
"""When the user explicitly sets
|
||||
``exclude_interactive_browser=False``, the SDK kwarg is set to
|
||||
False. Without the opt-in we don't pass the kwarg at all (SDK
|
||||
default is True / browser excluded)."""
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
|
||||
build_credential(EntraIdentityConfig(exclude_interactive_browser=False))
|
||||
kwargs = fake_azure_identity.last_credential_kwargs
|
||||
assert kwargs["exclude_interactive_browser_credential"] is False
|
||||
|
||||
def test_credential_is_cached_per_config(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
|
||||
cfg = EntraIdentityConfig(scope="s1")
|
||||
c1 = build_credential(cfg)
|
||||
c2 = build_credential(cfg)
|
||||
assert c1 is c2
|
||||
assert fake_azure_identity.credential_count == 1
|
||||
|
||||
def test_distinct_configs_get_distinct_credentials(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import EntraIdentityConfig, build_credential
|
||||
c1 = build_credential(EntraIdentityConfig(scope="s1"))
|
||||
c2 = build_credential(EntraIdentityConfig(scope="s2"))
|
||||
assert c1 is not c2
|
||||
assert fake_azure_identity.credential_count == 2
|
||||
|
||||
def test_reset_cache_invalidates(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import (
|
||||
EntraIdentityConfig,
|
||||
build_credential,
|
||||
reset_credential_cache,
|
||||
)
|
||||
cfg = EntraIdentityConfig(scope="x")
|
||||
c1 = build_credential(cfg)
|
||||
reset_credential_cache()
|
||||
c2 = build_credential(cfg)
|
||||
assert c1 is not c2
|
||||
|
||||
|
||||
class TestBuildTokenProvider:
|
||||
def test_returns_callable_for_scope(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import build_token_provider
|
||||
provider = build_token_provider(scope="https://ai.azure.com/.default")
|
||||
assert callable(provider)
|
||||
assert provider() == "jwt-for-https://ai.azure.com/.default"
|
||||
assert fake_azure_identity.last_scope == "https://ai.azure.com/.default"
|
||||
|
||||
def test_falls_back_to_default_scope_when_unspecified(self, fake_azure_identity):
|
||||
"""When neither ``scope`` nor ``config`` is provided,
|
||||
``build_token_provider`` uses ``SCOPE_AI_AZURE_DEFAULT`` —
|
||||
Microsoft's documented Foundry inference scope. ``base_url`` is
|
||||
accepted for back-compat but ignored."""
|
||||
from agent.azure_identity_adapter import (
|
||||
SCOPE_AI_AZURE_DEFAULT,
|
||||
build_token_provider,
|
||||
)
|
||||
build_token_provider(base_url="https://r.openai.azure.com/openai/v1")
|
||||
assert fake_azure_identity.last_scope == SCOPE_AI_AZURE_DEFAULT
|
||||
|
||||
def test_explicit_scope_wins_over_base_url(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import build_token_provider
|
||||
build_token_provider(
|
||||
scope="https://override.example/.default",
|
||||
base_url="https://r.openai.azure.com/openai/v1",
|
||||
)
|
||||
assert fake_azure_identity.last_scope == "https://override.example/.default"
|
||||
|
||||
def test_config_object_wins_over_kwargs(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import (
|
||||
EntraIdentityConfig,
|
||||
build_token_provider,
|
||||
)
|
||||
cfg = EntraIdentityConfig(scope="cfg-scope")
|
||||
build_token_provider(scope="ignored", config=cfg)
|
||||
assert fake_azure_identity.last_scope == "cfg-scope"
|
||||
assert fake_azure_identity.last_credential_kwargs == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy-install / missing-package surface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRequireAzureIdentityMissing:
|
||||
def test_clear_error_when_lazy_install_disabled(self, monkeypatch):
|
||||
"""When azure-identity isn't importable AND lazy installs are
|
||||
off, the adapter must raise ImportError with an actionable
|
||||
message, not propagate FeatureUnavailable."""
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
|
||||
# Force the import path to fail.
|
||||
original_import = __builtins__["__import__"] if isinstance(__builtins__, dict) else __import__
|
||||
def _fake_import(name, *args, **kwargs):
|
||||
if name == "azure.identity" or name.startswith("azure.identity."):
|
||||
raise ImportError("simulated missing azure-identity")
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr("builtins.__import__", _fake_import)
|
||||
|
||||
# Simulate lazy installs disabled.
|
||||
from tools.lazy_deps import FeatureUnavailable
|
||||
|
||||
def _fake_ensure(*args, **kwargs):
|
||||
raise FeatureUnavailable(
|
||||
"provider.azure_identity",
|
||||
("azure-identity==1.25.3",),
|
||||
"lazy installs disabled (test simulation)",
|
||||
)
|
||||
|
||||
# The adapter calls ``ensure`` from ``tools.lazy_deps``; intercept
|
||||
# it by patching the actual symbol path.
|
||||
monkeypatch.setattr("tools.lazy_deps.ensure", _fake_ensure)
|
||||
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
_adapter._require_azure_identity()
|
||||
msg = str(exc_info.value)
|
||||
assert "azure-identity" in msg
|
||||
assert "Foundry" in msg or "foundry" in msg.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# has_azure_identity_credentials probe (timeout-bounded)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHasAzureIdentityCredentials:
|
||||
def test_returns_false_when_package_missing_and_install_disabled(self, monkeypatch):
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
|
||||
assert _adapter.has_azure_identity_credentials(
|
||||
"https://x/.default", allow_install=False,
|
||||
) is False
|
||||
|
||||
def test_lazy_install_triggered_when_package_missing(self, monkeypatch):
|
||||
"""With allow_install=True (default), the probe must trigger the
|
||||
lazy-install path before bailing — otherwise the wizard's
|
||||
``preflight`` would silently fail for fresh installs that haven't
|
||||
run ``pip install azure-identity`` yet."""
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
|
||||
installed = {"called": False}
|
||||
|
||||
def _fake_install():
|
||||
installed["called"] = True
|
||||
# After install, pretend the package is now importable.
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
|
||||
return SimpleNamespace(
|
||||
DefaultAzureCredential=lambda **kw: SimpleNamespace(
|
||||
kwargs=kw,
|
||||
get_token=lambda scope: SimpleNamespace(token="post-install-jwt", expires_on=0),
|
||||
),
|
||||
get_bearer_token_provider=lambda c, s: lambda: "x",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
|
||||
monkeypatch.setattr(_adapter, "_require_azure_identity", _fake_install)
|
||||
|
||||
# Provide a credential factory so the probe proceeds after install.
|
||||
monkeypatch.setattr(
|
||||
_adapter, "build_credential",
|
||||
lambda config: SimpleNamespace(
|
||||
get_token=lambda scope: SimpleNamespace(token="probe-jwt", expires_on=0),
|
||||
),
|
||||
)
|
||||
|
||||
result = _adapter.has_azure_identity_credentials(
|
||||
"https://x/.default", timeout_seconds=0.5,
|
||||
)
|
||||
assert installed["called"] is True, (
|
||||
"has_azure_identity_credentials must trigger lazy install "
|
||||
"before bailing"
|
||||
)
|
||||
assert result is True
|
||||
|
||||
def test_returns_true_on_successful_token_mint(self, fake_azure_identity):
|
||||
from agent.azure_identity_adapter import has_azure_identity_credentials
|
||||
assert has_azure_identity_credentials("https://x/.default", timeout_seconds=0.5) is True
|
||||
|
||||
def test_returns_false_when_get_token_raises(self, monkeypatch):
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
|
||||
def _failing_credential(_config):
|
||||
class _Cred:
|
||||
def get_token(self, scope):
|
||||
raise RuntimeError("simulated chain exhaustion")
|
||||
return _Cred()
|
||||
|
||||
monkeypatch.setattr(_adapter, "build_credential", _failing_credential)
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
|
||||
assert _adapter.has_azure_identity_credentials("https://x/.default", timeout_seconds=0.5) is False
|
||||
|
||||
def test_returns_false_on_timeout(self, monkeypatch):
|
||||
"""Slow IMDS / network must time out, not hang the caller."""
|
||||
import threading
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
|
||||
slow_release = threading.Event()
|
||||
|
||||
def _slow_credential(_config):
|
||||
class _Cred:
|
||||
def get_token(self, scope):
|
||||
# Block forever from the test's perspective; the
|
||||
# adapter must give up via its thread-bounded probe.
|
||||
slow_release.wait(timeout=10)
|
||||
return SimpleNamespace(token="never-returned", expires_on=0)
|
||||
return _Cred()
|
||||
|
||||
monkeypatch.setattr(_adapter, "build_credential", _slow_credential)
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
|
||||
try:
|
||||
assert _adapter.has_azure_identity_credentials(
|
||||
"https://x/.default", timeout_seconds=0.1
|
||||
) is False
|
||||
finally:
|
||||
slow_release.set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# describe_active_credential — used by hermes doctor + hermes auth
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDescribeActiveCredential:
|
||||
def test_reports_not_installed(self, monkeypatch):
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
|
||||
info = _adapter.describe_active_credential(
|
||||
scope="https://x/.default", allow_install=False,
|
||||
)
|
||||
assert info["ok"] is False
|
||||
assert "not installed" in info["error"].lower()
|
||||
assert "pip install" in info["hint"].lower()
|
||||
|
||||
def test_reports_install_failure(self, monkeypatch):
|
||||
"""When lazy install is allowed but fails (e.g. lazy installs
|
||||
disabled), the diagnostic surfaces the failure as the error."""
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: False)
|
||||
|
||||
def _fail_install():
|
||||
raise ImportError("simulated: lazy installs disabled")
|
||||
|
||||
monkeypatch.setattr(_adapter, "_require_azure_identity", _fail_install)
|
||||
info = _adapter.describe_active_credential(
|
||||
scope="https://x/.default", allow_install=True,
|
||||
)
|
||||
assert info["ok"] is False
|
||||
assert "lazy installs disabled" in info["error"]
|
||||
assert "lazy" in info["hint"].lower()
|
||||
|
||||
def test_reports_env_sources_for_managed_identity(self, fake_azure_identity, monkeypatch):
|
||||
from agent.azure_identity_adapter import describe_active_credential
|
||||
monkeypatch.setenv("IDENTITY_ENDPOINT", "http://169.254.169.254")
|
||||
info = describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
|
||||
assert info["ok"] is True
|
||||
sources = info.get("env_sources") or []
|
||||
assert any("ManagedIdentity" in s for s in sources)
|
||||
|
||||
def test_reports_env_sources_for_workload_identity(self, fake_azure_identity, monkeypatch):
|
||||
from agent.azure_identity_adapter import describe_active_credential
|
||||
monkeypatch.setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/secrets/azure/federated-token")
|
||||
info = describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
|
||||
sources = info.get("env_sources") or []
|
||||
assert any("WorkloadIdentity" in s for s in sources)
|
||||
|
||||
def test_reports_env_sources_for_service_principal(self, fake_azure_identity, monkeypatch):
|
||||
from agent.azure_identity_adapter import describe_active_credential
|
||||
monkeypatch.setenv("AZURE_TENANT_ID", "t")
|
||||
monkeypatch.setenv("AZURE_CLIENT_ID", "c")
|
||||
monkeypatch.setenv("AZURE_CLIENT_SECRET", "s")
|
||||
info = describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
|
||||
sources = info.get("env_sources") or []
|
||||
assert any("EnvironmentCredential" in s for s in sources)
|
||||
|
||||
def test_reports_error_on_chain_failure(self, monkeypatch):
|
||||
from agent import azure_identity_adapter as _adapter
|
||||
|
||||
def _failing_credential(_config):
|
||||
class _Cred:
|
||||
def get_token(self, scope):
|
||||
raise RuntimeError("auth failed")
|
||||
return _Cred()
|
||||
|
||||
monkeypatch.setattr(_adapter, "build_credential", _failing_credential)
|
||||
monkeypatch.setattr(_adapter, "has_azure_identity_installed", lambda: True)
|
||||
info = _adapter.describe_active_credential(scope="https://x/.default", timeout_seconds=0.5)
|
||||
assert info["ok"] is False
|
||||
assert "auth failed" in info.get("error", "")
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Tests for the 1M-context beta header on AWS Bedrock Claude models.
|
||||
|
||||
Claude Opus 4.6/4.7 and Sonnet 4.6 support a 1M context window, but on AWS
|
||||
Bedrock (and Microsoft Foundry) that window is still gated behind the
|
||||
``context-1m-2025-08-07`` beta header as of 2026-04. Without it, Bedrock
|
||||
caps these models at 200K even though ``model_metadata.py`` advertises 1M.
|
||||
|
||||
These tests guard the invariant that the header is always emitted on the
|
||||
Bedrock client path, and that it survives the MiniMax bearer-auth strip.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestBedrockContext1MBeta:
|
||||
"""``context-1m-2025-08-07`` must reach Bedrock Claude requests."""
|
||||
|
||||
|
||||
|
||||
def test_common_betas_strips_1m_for_minimax(self):
|
||||
"""MiniMax bearer-auth endpoints host their own models — strip 1M beta."""
|
||||
from agent.anthropic_adapter import (
|
||||
_common_betas_for_base_url,
|
||||
_CONTEXT_1M_BETA,
|
||||
)
|
||||
|
||||
for url in (
|
||||
"https://api.minimax.io/anthropic",
|
||||
"https://api.minimaxi.com/anthropic",
|
||||
):
|
||||
betas = _common_betas_for_base_url(url)
|
||||
assert _CONTEXT_1M_BETA not in betas, (
|
||||
f"1M beta must be stripped for MiniMax bearer endpoint {url}"
|
||||
)
|
||||
# Other betas still present
|
||||
assert "interleaved-thinking-2025-05-14" in betas
|
||||
|
||||
def test_build_anthropic_bedrock_client_sends_1m_beta(self):
|
||||
"""AnthropicBedrock client must carry the 1M beta in default_headers.
|
||||
|
||||
This is the load-bearing assertion for the reported bug:
|
||||
without this header Bedrock serves Opus 4.6/4.7 with a 200K cap.
|
||||
"""
|
||||
import agent.anthropic_adapter as adapter
|
||||
|
||||
fake_sdk = MagicMock()
|
||||
fake_sdk.AnthropicBedrock = MagicMock()
|
||||
|
||||
with patch.object(adapter, "_anthropic_sdk", fake_sdk):
|
||||
adapter.build_anthropic_bedrock_client(region="us-west-2")
|
||||
|
||||
call_kwargs = fake_sdk.AnthropicBedrock.call_args.kwargs
|
||||
assert call_kwargs["aws_region"] == "us-west-2"
|
||||
|
||||
default_headers = call_kwargs.get("default_headers") or {}
|
||||
beta_header = default_headers.get("anthropic-beta", "")
|
||||
assert "context-1m-2025-08-07" in beta_header, (
|
||||
"Bedrock client must send context-1m-2025-08-07 or Opus 4.6/4.7 "
|
||||
"silently caps at 200K context"
|
||||
)
|
||||
# Other common betas still present — no regression.
|
||||
assert "interleaved-thinking-2025-05-14" in beta_header
|
||||
assert "fine-grained-tool-streaming-2025-05-14" in beta_header
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,592 @@
|
||||
"""Integration tests for the AWS Bedrock provider wiring.
|
||||
|
||||
Verifies that the Bedrock provider is correctly registered in the
|
||||
provider registry, model catalog, and runtime resolution pipeline.
|
||||
These tests do NOT require AWS credentials or boto3 — all AWS calls
|
||||
are mocked.
|
||||
|
||||
Note: Tests that import ``hermes_cli.auth`` or ``hermes_cli.runtime_provider``
|
||||
require Python 3.10+ due to ``str | None`` type syntax in the import chain.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestProviderRegistry:
|
||||
"""Verify Bedrock is registered in PROVIDER_REGISTRY."""
|
||||
|
||||
def test_bedrock_in_registry(self):
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
assert "bedrock" in PROVIDER_REGISTRY
|
||||
|
||||
def test_bedrock_auth_type_is_aws_sdk(self):
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
pconfig = PROVIDER_REGISTRY["bedrock"]
|
||||
assert pconfig.auth_type == "aws_sdk"
|
||||
|
||||
def test_bedrock_has_no_api_key_env_vars(self):
|
||||
"""Bedrock uses the AWS SDK credential chain, not API keys."""
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
pconfig = PROVIDER_REGISTRY["bedrock"]
|
||||
assert pconfig.api_key_env_vars == ()
|
||||
|
||||
def test_bedrock_base_url_env_var(self):
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
pconfig = PROVIDER_REGISTRY["bedrock"]
|
||||
assert pconfig.base_url_env_var == "BEDROCK_BASE_URL"
|
||||
|
||||
|
||||
class TestProviderAliases:
|
||||
"""Verify Bedrock aliases resolve correctly."""
|
||||
|
||||
def test_aws_alias(self):
|
||||
from hermes_cli.models import _PROVIDER_ALIASES
|
||||
assert _PROVIDER_ALIASES.get("aws") == "bedrock"
|
||||
|
||||
def test_aws_bedrock_alias(self):
|
||||
from hermes_cli.models import _PROVIDER_ALIASES
|
||||
assert _PROVIDER_ALIASES.get("aws-bedrock") == "bedrock"
|
||||
|
||||
def test_amazon_bedrock_alias(self):
|
||||
from hermes_cli.models import _PROVIDER_ALIASES
|
||||
assert _PROVIDER_ALIASES.get("amazon-bedrock") == "bedrock"
|
||||
|
||||
def test_amazon_alias(self):
|
||||
from hermes_cli.models import _PROVIDER_ALIASES
|
||||
assert _PROVIDER_ALIASES.get("amazon") == "bedrock"
|
||||
|
||||
|
||||
class TestProviderLabels:
|
||||
"""Verify Bedrock appears in provider labels."""
|
||||
|
||||
def test_bedrock_label(self):
|
||||
from hermes_cli.models import _PROVIDER_LABELS
|
||||
assert _PROVIDER_LABELS.get("bedrock") == "AWS Bedrock"
|
||||
|
||||
|
||||
class TestModelCatalog:
|
||||
"""Verify Bedrock has a static model fallback list."""
|
||||
|
||||
def test_bedrock_has_curated_models(self):
|
||||
from hermes_cli.models import _PROVIDER_MODELS
|
||||
models = _PROVIDER_MODELS.get("bedrock", [])
|
||||
assert len(models) > 0
|
||||
|
||||
def test_bedrock_models_include_claude(self):
|
||||
from hermes_cli.models import _PROVIDER_MODELS
|
||||
models = _PROVIDER_MODELS.get("bedrock", [])
|
||||
claude_models = [m for m in models if "anthropic.claude" in m]
|
||||
assert len(claude_models) > 0
|
||||
|
||||
def test_bedrock_models_include_nova(self):
|
||||
from hermes_cli.models import _PROVIDER_MODELS
|
||||
models = _PROVIDER_MODELS.get("bedrock", [])
|
||||
nova_models = [m for m in models if "amazon.nova" in m]
|
||||
assert len(nova_models) > 0
|
||||
|
||||
|
||||
class TestResolveProvider:
|
||||
"""Verify resolve_provider() handles bedrock correctly."""
|
||||
|
||||
def test_explicit_bedrock_resolves(self, monkeypatch):
|
||||
"""When user explicitly requests 'bedrock', it should resolve."""
|
||||
# bedrock is in the registry, so resolve_provider should return it
|
||||
from hermes_cli.auth import resolve_provider
|
||||
result = resolve_provider("bedrock")
|
||||
assert result == "bedrock"
|
||||
|
||||
def test_aws_alias_resolves_to_bedrock(self):
|
||||
from hermes_cli.auth import resolve_provider
|
||||
result = resolve_provider("aws")
|
||||
assert result == "bedrock"
|
||||
|
||||
def test_amazon_bedrock_alias_resolves(self):
|
||||
from hermes_cli.auth import resolve_provider
|
||||
result = resolve_provider("amazon-bedrock")
|
||||
assert result == "bedrock"
|
||||
|
||||
def test_auto_detect_with_aws_credentials(self, monkeypatch):
|
||||
"""When AWS credentials are present and no other provider is configured,
|
||||
auto-detect should find bedrock."""
|
||||
from hermes_cli.auth import resolve_provider
|
||||
|
||||
# Clear all other provider env vars
|
||||
for var in ["OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_TOKEN", "GOOGLE_API_KEY", "DEEPSEEK_API_KEY"]:
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
# Set AWS credentials
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
|
||||
# Mock the auth store to have no active provider
|
||||
with patch("hermes_cli.auth._load_auth_store", return_value={}):
|
||||
result = resolve_provider("auto")
|
||||
assert result == "bedrock"
|
||||
|
||||
|
||||
class TestRuntimeProvider:
|
||||
"""Verify resolve_runtime_provider() handles bedrock correctly."""
|
||||
|
||||
def test_bedrock_runtime_resolution(self, monkeypatch):
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
monkeypatch.setenv("AWS_REGION", "eu-west-1")
|
||||
|
||||
# Mock resolve_provider to return bedrock
|
||||
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
|
||||
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}):
|
||||
result = resolve_runtime_provider(requested="bedrock")
|
||||
|
||||
assert result["provider"] == "bedrock"
|
||||
assert result["api_mode"] == "bedrock_converse"
|
||||
assert result["region"] == "eu-west-1"
|
||||
assert "bedrock-runtime.eu-west-1.amazonaws.com" in result["base_url"]
|
||||
assert result["api_key"] == "aws-sdk"
|
||||
|
||||
def test_bedrock_runtime_default_region(self, monkeypatch):
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
|
||||
monkeypatch.setenv("AWS_PROFILE", "default")
|
||||
monkeypatch.delenv("AWS_REGION", raising=False)
|
||||
monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
|
||||
|
||||
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
|
||||
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}):
|
||||
result = resolve_runtime_provider(requested="bedrock")
|
||||
|
||||
assert result["region"] == "us-east-1"
|
||||
|
||||
def test_bedrock_runtime_no_credentials_raises_on_auto_detect(self, monkeypatch):
|
||||
"""When bedrock is auto-detected (not explicitly requested) and no
|
||||
credentials are found, runtime resolution should raise AuthError."""
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
from hermes_cli.auth import AuthError
|
||||
|
||||
# Clear all AWS env vars
|
||||
for var in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_PROFILE",
|
||||
"AWS_BEARER_TOKEN_BEDROCK", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
|
||||
"AWS_WEB_IDENTITY_TOKEN_FILE"]:
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
# Mock both the provider resolution and boto3's credential chain
|
||||
mock_session = MagicMock()
|
||||
mock_session.get_credentials.return_value = None
|
||||
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
|
||||
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}), \
|
||||
patch("hermes_cli.runtime_provider.resolve_requested_provider", return_value="auto"), \
|
||||
patch.dict("sys.modules", {"botocore": MagicMock(), "botocore.session": MagicMock()}):
|
||||
import botocore.session as _bs
|
||||
_bs.get_session = MagicMock(return_value=mock_session)
|
||||
with pytest.raises(AuthError, match="No AWS credentials"):
|
||||
resolve_runtime_provider(requested="auto")
|
||||
|
||||
def test_bedrock_runtime_explicit_skips_credential_check(self, monkeypatch):
|
||||
"""When user explicitly requests bedrock, trust boto3's credential chain
|
||||
even if env-var detection finds nothing (covers IMDS, SSO, etc.)."""
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
|
||||
# No AWS env vars set — but explicit bedrock request should not raise
|
||||
for var in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_PROFILE",
|
||||
"AWS_BEARER_TOKEN_BEDROCK"]:
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
with patch("hermes_cli.runtime_provider.resolve_provider", return_value="bedrock"), \
|
||||
patch("hermes_cli.runtime_provider._get_model_config", return_value={"provider": "bedrock"}):
|
||||
result = resolve_runtime_provider(requested="bedrock")
|
||||
assert result["provider"] == "bedrock"
|
||||
assert result["api_mode"] == "bedrock_converse"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# providers.py integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestProvidersModule:
|
||||
"""Verify bedrock is wired into hermes_cli/providers.py."""
|
||||
|
||||
def test_bedrock_alias_in_providers(self):
|
||||
from hermes_cli.providers import ALIASES
|
||||
assert ALIASES.get("bedrock") is None # "bedrock" IS the canonical name, not an alias
|
||||
assert ALIASES.get("aws") == "bedrock"
|
||||
assert ALIASES.get("aws-bedrock") == "bedrock"
|
||||
|
||||
def test_bedrock_transport_mapping(self):
|
||||
from hermes_cli.providers import TRANSPORT_TO_API_MODE
|
||||
assert TRANSPORT_TO_API_MODE.get("bedrock_converse") == "bedrock_converse"
|
||||
|
||||
def test_determine_api_mode_from_bedrock_url(self):
|
||||
from hermes_cli.providers import determine_api_mode
|
||||
assert determine_api_mode(
|
||||
"unknown", "https://bedrock-runtime.us-east-1.amazonaws.com"
|
||||
) == "bedrock_converse"
|
||||
|
||||
def test_label_override(self):
|
||||
from hermes_cli.providers import _LABEL_OVERRIDES
|
||||
assert _LABEL_OVERRIDES.get("bedrock") == "AWS Bedrock"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error classifier integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestErrorClassifierBedrock:
|
||||
"""Verify Bedrock error patterns are in the global error classifier."""
|
||||
|
||||
def test_throttling_in_rate_limit_patterns(self):
|
||||
from agent.error_classifier import _RATE_LIMIT_PATTERNS
|
||||
assert "throttlingexception" in _RATE_LIMIT_PATTERNS
|
||||
|
||||
def test_context_overflow_patterns(self):
|
||||
from agent.error_classifier import _CONTEXT_OVERFLOW_PATTERNS
|
||||
assert "input is too long" in _CONTEXT_OVERFLOW_PATTERNS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# pyproject.toml bedrock extra
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPackaging:
|
||||
"""Verify Bedrock remains a declared lazy optional dependency."""
|
||||
|
||||
@staticmethod
|
||||
def _optional_dependencies():
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
|
||||
return tomllib.loads(content)["project"]["optional-dependencies"]
|
||||
|
||||
def test_bedrock_extra_exists(self):
|
||||
extras = self._optional_dependencies()
|
||||
assert "bedrock" in extras
|
||||
assert any(dep.startswith("boto3==") for dep in extras["bedrock"])
|
||||
|
||||
def test_bedrock_is_not_eager_installed_by_all_extra(self):
|
||||
extras = self._optional_dependencies()
|
||||
assert "hermes-agent[bedrock]" not in extras["all"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model ID dot preservation — regression for #11976
|
||||
# ---------------------------------------------------------------------------
|
||||
# AWS Bedrock inference-profile model IDs embed structural dots:
|
||||
#
|
||||
# global.anthropic.claude-opus-4-7
|
||||
# us.anthropic.claude-sonnet-4-5-20250929-v1:0
|
||||
# apac.anthropic.claude-haiku-4-5
|
||||
#
|
||||
# ``agent.anthropic_adapter.normalize_model_name`` converts dots to hyphens
|
||||
# unless the caller opts in via ``preserve_dots=True``. Before this fix,
|
||||
# ``AIAgent._anthropic_preserve_dots`` returned False for the ``bedrock``
|
||||
# provider, so Claude-on-Bedrock requests went out with
|
||||
# ``global-anthropic-claude-opus-4-7`` (all dots mangled to hyphens) and
|
||||
# Bedrock rejected them with:
|
||||
#
|
||||
# HTTP 400: The provided model identifier is invalid.
|
||||
#
|
||||
# The fix adds ``bedrock`` to the preserve-dots provider allowlist and
|
||||
# ``bedrock-runtime.`` to the base-URL heuristic, mirroring the shape of
|
||||
# the opencode-go fix for #5211 (commit f77be22c), which extended this
|
||||
# same allowlist.
|
||||
|
||||
|
||||
class TestBedrockPreserveDotsFlag:
|
||||
"""``AIAgent._anthropic_preserve_dots`` must return True on Bedrock so
|
||||
inference-profile IDs survive the normalize step intact."""
|
||||
|
||||
def test_bedrock_provider_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="bedrock", base_url="")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_bedrock_runtime_us_east_1_url_preserves_dots(self):
|
||||
"""Defense-in-depth: even without an explicit ``provider="bedrock"``,
|
||||
a ``bedrock-runtime.us-east-1.amazonaws.com`` base URL must not
|
||||
mangle dots."""
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(
|
||||
provider="custom",
|
||||
base_url="https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
)
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_bedrock_runtime_ap_northeast_2_url_preserves_dots(self):
|
||||
"""Reporter-reported region (ap-northeast-2) exercises the same
|
||||
base-URL heuristic."""
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(
|
||||
provider="custom",
|
||||
base_url="https://bedrock-runtime.ap-northeast-2.amazonaws.com",
|
||||
)
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_non_bedrock_aws_url_does_not_preserve_dots(self):
|
||||
"""Unrelated AWS endpoints (e.g. ``s3.us-east-1.amazonaws.com``)
|
||||
must not accidentally activate the dot-preservation heuristic —
|
||||
the heuristic is scoped to the ``bedrock-runtime.`` substring
|
||||
specifically."""
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(
|
||||
provider="custom",
|
||||
base_url="https://s3.us-east-1.amazonaws.com",
|
||||
)
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is False
|
||||
|
||||
def test_anthropic_native_still_does_not_preserve_dots(self):
|
||||
"""Canary: adding Bedrock to the allowlist must not weaken the
|
||||
existing Anthropic native behaviour — ``claude-sonnet-4.6`` still
|
||||
becomes ``claude-sonnet-4-6`` for the Anthropic API."""
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="anthropic", base_url="https://api.anthropic.com")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is False
|
||||
|
||||
|
||||
class TestBedrockModelNameNormalization:
|
||||
"""End-to-end: ``normalize_model_name`` + the preserve-dots flag
|
||||
reproduce the exact production request shape for each Bedrock model
|
||||
family, confirming the fix resolves the reporter's HTTP 400."""
|
||||
|
||||
def test_global_anthropic_inference_profile_preserved(self):
|
||||
"""The reporter's exact model ID."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name(
|
||||
"global.anthropic.claude-opus-4-7", preserve_dots=True
|
||||
) == "global.anthropic.claude-opus-4-7"
|
||||
|
||||
def test_us_anthropic_dated_inference_profile_preserved(self):
|
||||
"""Regional + dated Sonnet inference profile."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name(
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
preserve_dots=True,
|
||||
) == "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
|
||||
def test_apac_anthropic_haiku_inference_profile_preserved(self):
|
||||
"""APAC inference profile — same structural-dot shape."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name(
|
||||
"apac.anthropic.claude-haiku-4-5", preserve_dots=True
|
||||
) == "apac.anthropic.claude-haiku-4-5"
|
||||
|
||||
def test_bedrock_prefix_preserved_without_preserve_dots(self):
|
||||
"""Bedrock inference profile IDs are auto-detected by prefix and
|
||||
always returned unmangled -- ``preserve_dots`` is irrelevant for
|
||||
these IDs because the dots are namespace separators, not version
|
||||
separators. Regression for #12295."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name(
|
||||
"global.anthropic.claude-opus-4-7", preserve_dots=False
|
||||
) == "global.anthropic.claude-opus-4-7"
|
||||
|
||||
def test_bare_foundation_model_id_preserved(self):
|
||||
"""Non-inference-profile Bedrock IDs
|
||||
(e.g. ``anthropic.claude-3-5-sonnet-20241022-v2:0``) use dots as
|
||||
vendor separators and must also survive intact under
|
||||
``preserve_dots=True``."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name(
|
||||
"anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
preserve_dots=True,
|
||||
) == "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
|
||||
|
||||
class TestBedrockBuildAnthropicKwargsEndToEnd:
|
||||
"""Integration: calling ``build_anthropic_kwargs`` with a Bedrock-
|
||||
shaped model ID and ``preserve_dots=True`` produces the unmangled
|
||||
model string in the outgoing kwargs — the exact body sent to the
|
||||
``bedrock-runtime.`` endpoint. This is the integration-level
|
||||
regression for the reporter's HTTP 400."""
|
||||
|
||||
def test_bedrock_inference_profile_survives_build_kwargs(self):
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="global.anthropic.claude-opus-4-7",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=None,
|
||||
max_tokens=1024,
|
||||
reasoning_config=None,
|
||||
preserve_dots=True,
|
||||
)
|
||||
assert kwargs["model"] == "global.anthropic.claude-opus-4-7", (
|
||||
"Bedrock inference-profile ID was mangled in build_anthropic_kwargs: "
|
||||
f"{kwargs['model']!r}"
|
||||
)
|
||||
|
||||
def test_bedrock_model_preserved_without_preserve_dots(self):
|
||||
"""Bedrock inference profile IDs survive ``build_anthropic_kwargs``
|
||||
even without ``preserve_dots=True`` -- the prefix auto-detection
|
||||
in ``normalize_model_name`` is the load-bearing piece.
|
||||
Regression for #12295."""
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="global.anthropic.claude-opus-4-7",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=None,
|
||||
max_tokens=1024,
|
||||
reasoning_config=None,
|
||||
preserve_dots=False,
|
||||
)
|
||||
assert kwargs["model"] == "global.anthropic.claude-opus-4-7"
|
||||
|
||||
|
||||
class TestBedrockModelIdDetection:
|
||||
"""Tests for ``_is_bedrock_model_id`` and the auto-detection that
|
||||
makes ``normalize_model_name`` preserve dots for Bedrock IDs
|
||||
regardless of ``preserve_dots``. Regression for #12295."""
|
||||
|
||||
def test_bare_bedrock_id_detected(self):
|
||||
from agent.anthropic_adapter import _is_bedrock_model_id
|
||||
assert _is_bedrock_model_id("anthropic.claude-opus-4-7") is True
|
||||
|
||||
def test_regional_us_prefix_detected(self):
|
||||
from agent.anthropic_adapter import _is_bedrock_model_id
|
||||
assert _is_bedrock_model_id("us.anthropic.claude-sonnet-4-5-v1:0") is True
|
||||
|
||||
def test_regional_global_prefix_detected(self):
|
||||
from agent.anthropic_adapter import _is_bedrock_model_id
|
||||
assert _is_bedrock_model_id("global.anthropic.claude-opus-4-7") is True
|
||||
|
||||
def test_regional_eu_prefix_detected(self):
|
||||
from agent.anthropic_adapter import _is_bedrock_model_id
|
||||
assert _is_bedrock_model_id("eu.anthropic.claude-sonnet-4-6") is True
|
||||
|
||||
def test_openrouter_format_not_detected(self):
|
||||
from agent.anthropic_adapter import _is_bedrock_model_id
|
||||
assert _is_bedrock_model_id("claude-opus-4.6") is False
|
||||
|
||||
def test_bare_claude_not_detected(self):
|
||||
from agent.anthropic_adapter import _is_bedrock_model_id
|
||||
assert _is_bedrock_model_id("claude-opus-4-7") is False
|
||||
|
||||
def test_bare_bedrock_id_preserved_without_flag(self):
|
||||
"""The primary bug from #12295: ``anthropic.claude-opus-4-7``
|
||||
sent to bedrock-mantle via auxiliary clients that don't pass
|
||||
``preserve_dots=True``."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name(
|
||||
"anthropic.claude-opus-4-7", preserve_dots=False
|
||||
) == "anthropic.claude-opus-4-7"
|
||||
|
||||
def test_openrouter_dots_still_converted(self):
|
||||
"""Non-Bedrock dotted model names must still be converted."""
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name("claude-opus-4.6") == "claude-opus-4-6"
|
||||
|
||||
def test_bare_bedrock_id_survives_build_kwargs(self):
|
||||
"""End-to-end: bare Bedrock ID through ``build_anthropic_kwargs``
|
||||
without ``preserve_dots=True`` -- the auxiliary client path."""
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="anthropic.claude-opus-4-7",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=None,
|
||||
max_tokens=1024,
|
||||
reasoning_config=None,
|
||||
preserve_dots=False,
|
||||
)
|
||||
assert kwargs["model"] == "anthropic.claude-opus-4-7"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# auxiliary_client Bedrock resolution — fix for #13919
|
||||
# ---------------------------------------------------------------------------
|
||||
# Before the fix, resolve_provider_client("bedrock", ...) fell through to the
|
||||
# "unhandled auth_type" warning and returned (None, None), breaking all
|
||||
# auxiliary tasks (compression, memory, summarization) for Bedrock users.
|
||||
|
||||
|
||||
class TestAuxiliaryClientBedrockResolution:
|
||||
"""Verify resolve_provider_client handles Bedrock's aws_sdk auth type."""
|
||||
|
||||
def test_bedrock_returns_client_with_credentials(self, monkeypatch):
|
||||
"""With valid AWS credentials, Bedrock should return a usable client."""
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
monkeypatch.setenv("AWS_REGION", "us-west-2")
|
||||
|
||||
mock_anthropic_bedrock = MagicMock()
|
||||
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
|
||||
return_value=mock_anthropic_bedrock):
|
||||
from agent.auxiliary_client import resolve_provider_client, AnthropicAuxiliaryClient
|
||||
client, model = resolve_provider_client("bedrock", None)
|
||||
|
||||
assert client is not None, (
|
||||
"resolve_provider_client('bedrock') returned None — "
|
||||
"aws_sdk auth type is not handled"
|
||||
)
|
||||
assert isinstance(client, AnthropicAuxiliaryClient)
|
||||
assert model is not None
|
||||
assert client.api_key == "aws-sdk"
|
||||
assert "us-west-2" in client.base_url
|
||||
|
||||
def test_bedrock_returns_none_without_credentials(self, monkeypatch):
|
||||
"""Without AWS credentials, Bedrock should return (None, None) gracefully."""
|
||||
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=False):
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, model = resolve_provider_client("bedrock", None)
|
||||
|
||||
assert client is None
|
||||
assert model is None
|
||||
|
||||
def test_bedrock_uses_configured_region(self, monkeypatch):
|
||||
"""Bedrock client base_url should reflect AWS_REGION."""
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
monkeypatch.setenv("AWS_REGION", "eu-central-1")
|
||||
|
||||
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
|
||||
return_value=MagicMock()):
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
client, _ = resolve_provider_client("bedrock", None)
|
||||
|
||||
assert client is not None
|
||||
assert "eu-central-1" in client.base_url
|
||||
|
||||
def test_bedrock_respects_explicit_model(self, monkeypatch):
|
||||
"""When caller passes an explicit model, it should be used."""
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
|
||||
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
|
||||
return_value=MagicMock()):
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
_, model = resolve_provider_client(
|
||||
"bedrock", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
)
|
||||
|
||||
assert "claude-sonnet" in model
|
||||
|
||||
def test_bedrock_async_mode(self, monkeypatch):
|
||||
"""Async mode should return an AsyncAnthropicAuxiliaryClient."""
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
|
||||
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
|
||||
return_value=MagicMock()):
|
||||
from agent.auxiliary_client import resolve_provider_client, AsyncAnthropicAuxiliaryClient
|
||||
client, model = resolve_provider_client("bedrock", None, async_mode=True)
|
||||
|
||||
assert client is not None
|
||||
assert isinstance(client, AsyncAnthropicAuxiliaryClient)
|
||||
|
||||
def test_bedrock_default_model_is_haiku(self, monkeypatch):
|
||||
"""Default auxiliary model for Bedrock should be Haiku (fast, cheap)."""
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
||||
|
||||
with patch("agent.anthropic_adapter.build_anthropic_bedrock_client",
|
||||
return_value=MagicMock()):
|
||||
from agent.auxiliary_client import resolve_provider_client
|
||||
_, model = resolve_provider_client("bedrock", None)
|
||||
|
||||
assert "haiku" in model.lower()
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Regression guard: Codex Cloudflare 403 mitigation headers.
|
||||
|
||||
The ``chatgpt.com/backend-api/codex`` endpoint sits behind a Cloudflare layer
|
||||
that whitelists a small set of first-party originators (``codex_cli_rs``,
|
||||
``codex_vscode``, ``codex_sdk_ts``, ``Codex*``). Requests from non-residential
|
||||
IPs (VPS, always-on servers, some corporate egress) that don't advertise an
|
||||
allowed originator are served 403 with ``cf-mitigated: challenge`` regardless
|
||||
of auth correctness.
|
||||
|
||||
``_codex_cloudflare_headers`` in ``agent.auxiliary_client`` centralizes the
|
||||
header set so the primary chat client (``run_agent.AIAgent.__init__`` +
|
||||
``_apply_client_headers_for_base_url``) and the auxiliary client paths
|
||||
(``_build_codex_client`` and the ``raw_codex`` branch of ``resolve_provider_client``)
|
||||
all emit the same headers.
|
||||
|
||||
These tests pin:
|
||||
- the originator value (must be ``codex_cli_rs`` — the whitelisted one)
|
||||
- the User-Agent shape (codex_cli_rs-prefixed)
|
||||
- ``ChatGPT-Account-ID`` extraction from the OAuth JWT (canonical casing,
|
||||
from codex-rs ``auth.rs``)
|
||||
- graceful handling of malformed tokens (drop the account-ID header, don't
|
||||
raise)
|
||||
- primary-client wiring at both entry points in ``run_agent.py``
|
||||
- aux-client wiring at both entry points in ``agent/auxiliary_client.py``
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_codex_jwt(account_id: str = "acct-test-123") -> str:
|
||||
"""Build a syntactically valid Codex-style JWT with the account_id claim."""
|
||||
def b64url(data: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(data).rstrip(b"=").decode()
|
||||
header = b64url(b'{"alg":"RS256","typ":"JWT"}')
|
||||
claims = {
|
||||
"sub": "user-xyz",
|
||||
"exp": 9999999999,
|
||||
"https://api.openai.com/auth": {
|
||||
"chatgpt_account_id": account_id,
|
||||
"chatgpt_plan_type": "plus",
|
||||
},
|
||||
}
|
||||
payload = b64url(json.dumps(claims).encode())
|
||||
sig = b64url(b"fake-sig")
|
||||
return f"{header}.{payload}.{sig}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _codex_cloudflare_headers — the shared helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCodexCloudflareHeaders:
|
||||
def test_originator_is_codex_cli_rs(self):
|
||||
"""Cloudflare whitelists codex_cli_rs — any other value is 403'd."""
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
headers = _codex_cloudflare_headers(_make_codex_jwt())
|
||||
assert headers["originator"] == "codex_cli_rs"
|
||||
|
||||
def test_user_agent_advertises_codex_cli_rs(self):
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
headers = _codex_cloudflare_headers(_make_codex_jwt())
|
||||
assert headers["User-Agent"].startswith("codex_cli_rs/")
|
||||
|
||||
def test_account_id_extracted_from_jwt(self):
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
headers = _codex_cloudflare_headers(_make_codex_jwt("acct-abc-999"))
|
||||
# Canonical casing — matches codex-rs auth.rs
|
||||
assert headers["ChatGPT-Account-ID"] == "acct-abc-999"
|
||||
|
||||
def test_canonical_header_casing(self):
|
||||
"""Upstream codex-rs uses PascalCase with trailing -ID. Match exactly."""
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
headers = _codex_cloudflare_headers(_make_codex_jwt())
|
||||
assert "ChatGPT-Account-ID" in headers
|
||||
# The lowercase/titlecase variants MUST NOT be used — pin to be explicit
|
||||
assert "chatgpt-account-id" not in headers
|
||||
assert "ChatGPT-Account-Id" not in headers
|
||||
|
||||
def test_malformed_token_drops_account_id_without_raising(self):
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
for bad in ["not-a-jwt", "", "only.one", " ", "...."]:
|
||||
headers = _codex_cloudflare_headers(bad)
|
||||
# Still returns base headers — never raises
|
||||
assert headers["originator"] == "codex_cli_rs"
|
||||
assert "ChatGPT-Account-ID" not in headers
|
||||
|
||||
def test_non_string_token_handled(self):
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
headers = _codex_cloudflare_headers(None) # type: ignore[arg-type]
|
||||
assert headers["originator"] == "codex_cli_rs"
|
||||
assert "ChatGPT-Account-ID" not in headers
|
||||
|
||||
def test_jwt_without_chatgpt_account_id_claim(self):
|
||||
"""A valid JWT that lacks the account_id claim should still return headers."""
|
||||
from agent.auxiliary_client import _codex_cloudflare_headers
|
||||
import base64 as _b64, json as _json
|
||||
|
||||
def b64url(data: bytes) -> str:
|
||||
return _b64.urlsafe_b64encode(data).rstrip(b"=").decode()
|
||||
payload = b64url(_json.dumps({"sub": "user-xyz", "exp": 9999999999}).encode())
|
||||
token = f"{b64url(b'{}')}.{payload}.{b64url(b'sig')}"
|
||||
headers = _codex_cloudflare_headers(token)
|
||||
assert headers["originator"] == "codex_cli_rs"
|
||||
assert "ChatGPT-Account-ID" not in headers
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Primary chat client wiring (run_agent.AIAgent)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPrimaryClientWiring:
|
||||
def test_init_wires_codex_headers_for_chatgpt_base_url(self):
|
||||
from run_agent import AIAgent
|
||||
token = _make_codex_jwt("acct-primary-init")
|
||||
with patch("run_agent.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
AIAgent(
|
||||
api_key=token,
|
||||
base_url="https://chatgpt.com/backend-api/codex",
|
||||
provider="openai-codex",
|
||||
model="gpt-5.4",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
|
||||
assert headers.get("originator") == "codex_cli_rs"
|
||||
assert headers.get("ChatGPT-Account-ID") == "acct-primary-init"
|
||||
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
|
||||
|
||||
def test_apply_client_headers_on_base_url_change(self):
|
||||
"""Credential-rotation / base-url change path must also emit codex headers."""
|
||||
from run_agent import AIAgent
|
||||
token = _make_codex_jwt("acct-rotation")
|
||||
with patch("run_agent.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
agent = AIAgent(
|
||||
api_key="placeholder-openrouter-key",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
provider="openrouter",
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
# Simulate rotation into a Codex credential
|
||||
agent._client_kwargs["api_key"] = token
|
||||
agent._apply_client_headers_for_base_url(
|
||||
"https://chatgpt.com/backend-api/codex"
|
||||
)
|
||||
headers = agent._client_kwargs.get("default_headers") or {}
|
||||
assert headers.get("originator") == "codex_cli_rs"
|
||||
assert headers.get("ChatGPT-Account-ID") == "acct-rotation"
|
||||
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
|
||||
|
||||
def test_apply_client_headers_clears_codex_headers_off_chatgpt(self):
|
||||
"""Switching AWAY from chatgpt.com must drop the codex headers."""
|
||||
from run_agent import AIAgent
|
||||
token = _make_codex_jwt()
|
||||
with patch("run_agent.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
agent = AIAgent(
|
||||
api_key=token,
|
||||
base_url="https://chatgpt.com/backend-api/codex",
|
||||
provider="openai-codex",
|
||||
model="gpt-5.4",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
# Sanity: headers are set initially
|
||||
assert "originator" in (agent._client_kwargs.get("default_headers") or {})
|
||||
agent._apply_client_headers_for_base_url(
|
||||
"https://api.anthropic.com"
|
||||
)
|
||||
# default_headers should be popped for anthropic base
|
||||
assert "default_headers" not in agent._client_kwargs
|
||||
|
||||
def test_openrouter_base_url_does_not_get_codex_headers(self):
|
||||
from run_agent import AIAgent
|
||||
with patch("run_agent.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
AIAgent(
|
||||
api_key="sk-or-test",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
provider="openrouter",
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
|
||||
assert headers.get("originator") != "codex_cli_rs"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auxiliary client wiring (agent.auxiliary_client)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestAuxiliaryClientWiring:
|
||||
def test_build_codex_client_passes_codex_headers(self, monkeypatch):
|
||||
"""_build_codex_client builds the OpenAI client used for compression /
|
||||
vision / title generation when routed through Codex. Must emit codex
|
||||
headers."""
|
||||
from agent import auxiliary_client
|
||||
token = _make_codex_jwt("acct-aux-try-codex")
|
||||
|
||||
# Force _select_pool_entry to return "no pool" so we fall through to
|
||||
# _read_codex_access_token.
|
||||
monkeypatch.setattr(
|
||||
auxiliary_client, "_select_pool_entry",
|
||||
lambda provider: (False, None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
auxiliary_client, "_read_codex_access_token",
|
||||
lambda: token,
|
||||
)
|
||||
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
client, model = auxiliary_client._build_codex_client("gpt-5.4")
|
||||
assert client is not None
|
||||
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
|
||||
assert headers.get("originator") == "codex_cli_rs"
|
||||
assert headers.get("ChatGPT-Account-ID") == "acct-aux-try-codex"
|
||||
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
|
||||
|
||||
def test_resolve_provider_client_raw_codex_passes_codex_headers(self, monkeypatch):
|
||||
"""The ``raw_codex=True`` branch (used by the main agent loop for direct
|
||||
responses.stream() access) must also emit codex headers."""
|
||||
from agent import auxiliary_client
|
||||
token = _make_codex_jwt("acct-aux-raw-codex")
|
||||
monkeypatch.setattr(
|
||||
auxiliary_client, "_read_codex_access_token",
|
||||
lambda: token,
|
||||
)
|
||||
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
|
||||
mock_openai.return_value = MagicMock()
|
||||
client, model = auxiliary_client.resolve_provider_client(
|
||||
"openai-codex", model="gpt-5.4", raw_codex=True,
|
||||
)
|
||||
assert client is not None
|
||||
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
|
||||
assert headers.get("originator") == "codex_cli_rs"
|
||||
assert headers.get("ChatGPT-Account-ID") == "acct-aux-raw-codex"
|
||||
assert headers.get("User-Agent", "").startswith("codex_cli_rs/")
|
||||
@@ -0,0 +1,176 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.codex_responses_adapter import (
|
||||
_format_responses_error,
|
||||
_normalize_codex_response,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_codex_response_drops_transient_rs_tmp_reasoning_items():
|
||||
response = SimpleNamespace(
|
||||
status="completed",
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="reasoning",
|
||||
id="rs_tmp_123",
|
||||
encrypted_content="opaque-transient",
|
||||
summary=[],
|
||||
),
|
||||
SimpleNamespace(
|
||||
type="reasoning",
|
||||
id="rs_456",
|
||||
encrypted_content="opaque-stable",
|
||||
summary=[SimpleNamespace(text="stable summary")],
|
||||
),
|
||||
SimpleNamespace(
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[SimpleNamespace(type="output_text", text="done")],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
assistant_message, finish_reason = _normalize_codex_response(response)
|
||||
|
||||
assert finish_reason == "stop"
|
||||
assert assistant_message.content == "done"
|
||||
assert assistant_message.codex_reasoning_items == [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"encrypted_content": "opaque-stable",
|
||||
"id": "rs_456",
|
||||
"summary": [{"type": "summary_text", "text": "stable summary"}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_normalize_codex_response_treats_summary_only_reasoning_as_incomplete():
|
||||
response = SimpleNamespace(
|
||||
status="completed",
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="reasoning",
|
||||
id="rs_tmp_789",
|
||||
encrypted_content="opaque-transient",
|
||||
summary=[SimpleNamespace(text="still thinking")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assistant_message, finish_reason = _normalize_codex_response(response)
|
||||
|
||||
assert finish_reason == "incomplete"
|
||||
assert assistant_message.content == ""
|
||||
assert assistant_message.reasoning == "still thinking"
|
||||
assert assistant_message.codex_reasoning_items is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _format_responses_error — adapted from anomalyco/opencode#28757.
|
||||
# Provider failures should surface BOTH the code (rate_limit_exceeded /
|
||||
# context_length_exceeded / internal_error / server_error) and the message,
|
||||
# so consumers can tell rate limits apart from context-length failures and
|
||||
# both apart from generic stream drops.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_format_responses_error_combines_code_and_message():
|
||||
err = {"code": "rate_limit_exceeded", "message": "Slow down"}
|
||||
assert _format_responses_error(err, "failed") == "rate_limit_exceeded: Slow down"
|
||||
|
||||
|
||||
def test_format_responses_error_message_only():
|
||||
err = {"message": "Upstream model unavailable"}
|
||||
assert _format_responses_error(err, "failed") == "Upstream model unavailable"
|
||||
|
||||
|
||||
def test_format_responses_error_code_only_when_message_empty():
|
||||
# Some providers/proxies emit a code with an empty message body. We
|
||||
# used to fall back to ``str(error_obj)`` — a dict dump — which leaked
|
||||
# ``{'code': 'internal_error', 'message': ''}`` into chat output. Now
|
||||
# the bare code is surfaced, which is the meaningful field.
|
||||
err = {"code": "internal_error", "message": ""}
|
||||
assert _format_responses_error(err, "failed") == "internal_error"
|
||||
|
||||
|
||||
def test_format_responses_error_code_only_when_message_missing():
|
||||
err = {"code": "server_error"}
|
||||
assert _format_responses_error(err, "failed") == "server_error"
|
||||
|
||||
|
||||
def test_format_responses_error_attribute_style_payload():
|
||||
# SDK objects expose ``code``/``message`` as attributes rather than dict
|
||||
# keys. The helper must accept both shapes since the Responses SDK
|
||||
# returns SimpleNamespace-style objects on ``response.failed``.
|
||||
err = SimpleNamespace(code="context_length_exceeded", message="too long")
|
||||
assert _format_responses_error(err, "failed") == "context_length_exceeded: too long"
|
||||
|
||||
|
||||
def test_format_responses_error_falls_back_to_status_when_empty():
|
||||
assert (
|
||||
_format_responses_error(None, "failed")
|
||||
== "Responses API returned status 'failed'"
|
||||
)
|
||||
assert (
|
||||
_format_responses_error(None, "cancelled")
|
||||
== "Responses API returned status 'cancelled'"
|
||||
)
|
||||
|
||||
|
||||
def test_format_responses_error_stringifies_opaque_payload():
|
||||
# Last-resort: a provider sent something that isn't a dict and has no
|
||||
# code/message attributes. Surface its repr rather than swallow it
|
||||
# silently — at least it's visible in logs.
|
||||
assert _format_responses_error("opaque sentinel", "failed") == "opaque sentinel"
|
||||
|
||||
|
||||
def test_format_responses_error_ignores_non_string_code_message():
|
||||
# Defensive: a malformed gateway could send numbers/objects in these
|
||||
# fields. We don't want to crash; we want a best-effort string.
|
||||
err = {"code": 500, "message": None}
|
||||
assert _format_responses_error(err, "failed") == "500"
|
||||
|
||||
|
||||
def test_normalize_codex_response_failed_includes_code_in_error():
|
||||
"""Regression: response_status == 'failed' should surface the error
|
||||
code, not just the message. Used to leak a bare 'Slow down' string
|
||||
that was indistinguishable from a generic stream truncation."""
|
||||
# ``output`` non-empty so we don't trip the "no output items" guard
|
||||
# before reaching the failed-status branch. Real failed responses
|
||||
# often DO carry a partial message item alongside the error.
|
||||
response = SimpleNamespace(
|
||||
status="failed",
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="incomplete",
|
||||
content=[SimpleNamespace(type="output_text", text="partial")],
|
||||
),
|
||||
],
|
||||
error={"code": "rate_limit_exceeded", "message": "Slow down"},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=r"^rate_limit_exceeded: Slow down$"):
|
||||
_normalize_codex_response(response)
|
||||
|
||||
|
||||
def test_normalize_codex_response_failed_with_message_only():
|
||||
"""Backwards-compat: a failed response with only a message field
|
||||
(no code) should still surface that message verbatim."""
|
||||
response = SimpleNamespace(
|
||||
status="failed",
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="incomplete",
|
||||
content=[SimpleNamespace(type="output_text", text="partial")],
|
||||
),
|
||||
],
|
||||
error={"message": "model error"},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=r"^model error$"):
|
||||
_normalize_codex_response(response)
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Regression tests for the Codex time-to-first-byte (TTFB) watchdog.
|
||||
|
||||
The chatgpt.com/backend-api/codex endpoint has an intermittent failure mode
|
||||
where it accepts the connection but never emits a single stream event. The
|
||||
watchdog in ``interruptible_api_call`` kills such a connection at a short TTFB
|
||||
cutoff (instead of waiting out the much longer wall-clock stale timeout) so the
|
||||
retry loop can reconnect promptly. Once any stream event arrives, the TTFB
|
||||
watchdog is satisfied and a separate idle watchdog handles streams that stop
|
||||
emitting SSE events.
|
||||
|
||||
The "bytes flowing" signal is ``agent._codex_stream_last_event_ts``, set on
|
||||
*any* event by ``codex_runtime.run_codex_stream`` — so reasoning-only or
|
||||
tool-call-only turns (which emit no output-text deltas) are not mistaken for a
|
||||
stall.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
# Stub optional heavy imports so run_agent imports cleanly in isolation.
|
||||
sys.modules.setdefault("fire", types.SimpleNamespace(Fire=lambda *a, **k: None))
|
||||
sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
|
||||
sys.modules.setdefault("fal_client", types.SimpleNamespace())
|
||||
|
||||
|
||||
def _make_codex_agent(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
(tmp_path / "config.yaml").write_text("{}\n", encoding="utf-8")
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
model="gpt-5.5",
|
||||
provider="openai-codex",
|
||||
api_key="sk-dummy",
|
||||
base_url="https://chatgpt.com/backend-api/codex",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
platform="cli",
|
||||
)
|
||||
# The watchdog is gated on the codex_responses api_mode; assert/force it so
|
||||
# the test is robust to detection-logic changes elsewhere.
|
||||
agent.api_mode = "codex_responses"
|
||||
monkeypatch.setattr(agent, "_emit_status", lambda *a, **k: None)
|
||||
# Keep the wall-clock stale timeout high so any early kill is unambiguously
|
||||
# the TTFB path, not the stale-call path.
|
||||
monkeypatch.setattr(
|
||||
agent, "_compute_non_stream_stale_timeout", lambda *a, **k: 60.0
|
||||
)
|
||||
return agent
|
||||
|
||||
|
||||
def test_ttfb_kills_when_no_stream_event(tmp_path, monkeypatch):
|
||||
"""Backend accepts the connection but emits no event -> killed at the TTFB
|
||||
cutoff, well before the 60s wall-clock stale timeout, with a retryable
|
||||
TimeoutError and a ``codex_ttfb_kill`` close reason."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
stop = {"flag": False}
|
||||
|
||||
def fake_hang(api_kwargs, client=None, on_first_delta=None):
|
||||
# Never set _codex_stream_last_event_ts: simulate zero events arriving.
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
|
||||
time.sleep(0.02)
|
||||
raise RuntimeError("connection closed")
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
with pytest.raises(TimeoutError) as excinfo:
|
||||
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
|
||||
elapsed = time.time() - t0
|
||||
assert "TTFB" in str(excinfo.value)
|
||||
assert "codex_ttfb_kill" in closes
|
||||
# ~1s cutoff + 2s join grace; must be far under the 60s stale timeout.
|
||||
assert elapsed < 15, f"TTFB watchdog took {elapsed:.1f}s"
|
||||
finally:
|
||||
stop["flag"] = True
|
||||
|
||||
|
||||
def test_ttfb_default_tolerates_slow_first_event(tmp_path, monkeypatch):
|
||||
"""With no env var set, the no-byte TTFB default is generous (120s), so a
|
||||
request whose first stream event is merely slow (~2s of backend admission /
|
||||
prefill) is NOT killed. This is the subscription-backed Codex case the tight
|
||||
12s default used to abort mid-prefill."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
# Default behavior: no explicit TTFB override.
|
||||
monkeypatch.delenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", raising=False)
|
||||
monkeypatch.delenv("HERMES_CODEX_TTFB_MAX_SECONDS", raising=False)
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
sentinel = SimpleNamespace(ok=True)
|
||||
|
||||
def fake_slow_first_event(api_kwargs, client=None, on_first_delta=None):
|
||||
# Backend is alive but slow to admit: first event lands after ~2s,
|
||||
# well under the 120s default cutoff. Mark the first byte so the
|
||||
# no-byte detector sees activity, then return the response.
|
||||
time.sleep(2.0)
|
||||
agent._codex_stream_last_event_ts = time.time()
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_slow_first_event)
|
||||
|
||||
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
|
||||
assert resp is sentinel
|
||||
assert "codex_ttfb_kill" not in closes
|
||||
|
||||
|
||||
def test_ttfb_includes_silent_hang_hint_for_gpt_5_5(tmp_path, monkeypatch):
|
||||
"""The no-first-byte watchdog should surface the same actionable hint as the
|
||||
stale-call timeout path when the model matches the silent-hang heuristic."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
|
||||
|
||||
closes: list = []
|
||||
statuses: list[str] = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(agent, "_buffer_status", lambda msg: statuses.append(msg))
|
||||
monkeypatch.setattr(agent, "_emit_status", lambda msg: statuses.append(msg))
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
stop = {"flag": False}
|
||||
|
||||
def fake_hang(api_kwargs, client=None, on_first_delta=None):
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
|
||||
time.sleep(0.02)
|
||||
raise RuntimeError("connection closed")
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
|
||||
|
||||
try:
|
||||
with pytest.raises(TimeoutError) as excinfo:
|
||||
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
|
||||
message = str(excinfo.value)
|
||||
assert "gpt-5.4" in message
|
||||
assert "gpt-5.3-codex" in message
|
||||
assert "gpt-5.4-codex" in message
|
||||
assert "codex_ttfb_kill" in closes
|
||||
assert statuses, "expected a user-facing watchdog status"
|
||||
assert any("gpt-5.4" in s and "gpt-5.3-codex" in s for s in statuses)
|
||||
finally:
|
||||
stop["flag"] = True
|
||||
|
||||
|
||||
def test_ttfb_high_env_is_capped_for_openai_codex(tmp_path, monkeypatch):
|
||||
"""A stale local env value like 90s must not make openai-codex wait 90s
|
||||
before reconnecting when the backend emits no SSE frames."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "90")
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_MAX_SECONDS", "1")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
stop = {"flag": False}
|
||||
|
||||
def fake_hang(api_kwargs, client=None, on_first_delta=None):
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
|
||||
time.sleep(0.02)
|
||||
raise RuntimeError("connection closed")
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
with pytest.raises(TimeoutError) as excinfo:
|
||||
h.interruptible_api_call(agent, {"model": "gpt-5.4", "input": "hi"})
|
||||
elapsed = time.time() - t0
|
||||
assert "TTFB threshold: 1s" in str(excinfo.value)
|
||||
assert "codex_ttfb_kill" in closes
|
||||
assert elapsed < 15, f"TTFB watchdog ignored cap and took {elapsed:.1f}s"
|
||||
finally:
|
||||
stop["flag"] = True
|
||||
|
||||
|
||||
def test_ttfb_does_not_kill_when_events_flow(tmp_path, monkeypatch):
|
||||
"""Once a stream event has arrived, a generation that runs past the TTFB
|
||||
cutoff is NOT killed by the watchdog — it completes normally."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
sentinel = SimpleNamespace(ok=True)
|
||||
|
||||
def fake_stream(api_kwargs, client=None, on_first_delta=None):
|
||||
# Bytes flowing: mark stream activity right away, then keep generating
|
||||
# past the 1s TTFB cutoff before returning a real response.
|
||||
agent._codex_stream_last_event_ts = time.time()
|
||||
if on_first_delta:
|
||||
on_first_delta()
|
||||
time.sleep(2.0)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
|
||||
|
||||
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
|
||||
assert resp is sentinel
|
||||
assert "codex_ttfb_kill" not in closes
|
||||
|
||||
|
||||
def test_event_idle_kills_after_first_event_then_silence(tmp_path, monkeypatch):
|
||||
"""If Codex emits an opening SSE event and then goes silent, kill it via
|
||||
the stream-idle watchdog instead of waiting for the long non-stream stale
|
||||
timeout."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "10")
|
||||
monkeypatch.setenv("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", "1")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent,
|
||||
"_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent,
|
||||
"_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
stop = {"flag": False}
|
||||
|
||||
def fake_stream(api_kwargs, client=None, on_first_delta=None):
|
||||
agent._codex_stream_last_event_ts = time.time()
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
|
||||
time.sleep(0.02)
|
||||
raise RuntimeError("connection closed")
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
|
||||
|
||||
try:
|
||||
with pytest.raises(TimeoutError) as excinfo:
|
||||
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
|
||||
assert "after first byte" in str(excinfo.value)
|
||||
assert "codex_stream_idle_kill" in closes
|
||||
assert "codex_ttfb_kill" not in closes
|
||||
finally:
|
||||
stop["flag"] = True
|
||||
|
||||
|
||||
def test_ttfb_disabled_via_env_zero(tmp_path, monkeypatch):
|
||||
"""Setting HERMES_CODEX_TTFB_TIMEOUT_SECONDS=0 disables the TTFB watchdog;
|
||||
a no-event stall then falls through to the (here, 60s) stale timeout, so a
|
||||
short hang is NOT killed by TTFB."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "0")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client",
|
||||
lambda c, reason=None: closes.append(reason),
|
||||
)
|
||||
|
||||
sentinel = SimpleNamespace(ok=True)
|
||||
|
||||
def fake_stream(api_kwargs, client=None, on_first_delta=None):
|
||||
# No event marker, but only briefly — well under the 60s stale timeout.
|
||||
time.sleep(2.0)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
|
||||
|
||||
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
|
||||
assert resp is sentinel
|
||||
assert "codex_ttfb_kill" not in closes
|
||||
|
||||
|
||||
def test_large_codex_request_waits_instead_of_ttfb_reconnect(tmp_path, monkeypatch):
|
||||
"""Large Codex inputs can legitimately take longer than the small-request
|
||||
first-byte cutoff before the first SSE frame. Preserve the full input and
|
||||
wait instead of killing/retrying at TTFB."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client", lambda c, reason=None: closes.append(reason)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client", lambda c, reason=None: closes.append(reason)
|
||||
)
|
||||
|
||||
sentinel = SimpleNamespace(ok=True)
|
||||
|
||||
def fake_stream(api_kwargs, client=None, on_first_delta=None):
|
||||
# No event marker for 2s: this would trip the 1s TTFB watchdog on a
|
||||
# small request, but should be allowed for a large request.
|
||||
time.sleep(2.0)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
|
||||
|
||||
large_input = "x" * 120_000 # ~30k estimated tokens, above large-request gate.
|
||||
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": large_input})
|
||||
assert resp is sentinel
|
||||
assert "codex_ttfb_kill" not in closes
|
||||
|
||||
|
||||
def test_large_codex_request_strict_ttfb_env_still_reconnects(tmp_path, monkeypatch):
|
||||
"""Operators can force the old early-reconnect behavior for large inputs
|
||||
with HERMES_CODEX_TTFB_STRICT=1."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
|
||||
monkeypatch.setenv("HERMES_CODEX_TTFB_STRICT", "1")
|
||||
|
||||
closes: list = []
|
||||
dummy_client = SimpleNamespace()
|
||||
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
|
||||
monkeypatch.setattr(
|
||||
agent, "_abort_request_openai_client", lambda c, reason=None: closes.append(reason)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent, "_close_request_openai_client", lambda c, reason=None: closes.append(reason)
|
||||
)
|
||||
|
||||
stop = {"flag": False}
|
||||
|
||||
def fake_hang(api_kwargs, client=None, on_first_delta=None):
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
|
||||
time.sleep(0.02)
|
||||
raise RuntimeError("connection closed")
|
||||
|
||||
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
|
||||
|
||||
large_input = "x" * 120_000
|
||||
try:
|
||||
with pytest.raises(TimeoutError) as excinfo:
|
||||
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": large_input})
|
||||
assert "TTFB threshold: 1s" in str(excinfo.value)
|
||||
assert "codex_ttfb_kill" in closes
|
||||
finally:
|
||||
stop["flag"] = True
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Tests for focus_topic flowing through the compressor.
|
||||
|
||||
Verifies that _generate_summary and compress accept and use the focus_topic
|
||||
parameter correctly. Inspired by Claude Code's /compact <focus>.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
|
||||
def _make_compressor():
|
||||
"""Create a ContextCompressor with minimal state for testing."""
|
||||
compressor = ContextCompressor.__new__(ContextCompressor)
|
||||
compressor.protect_first_n = 2
|
||||
compressor.protect_last_n = 5
|
||||
compressor.tail_token_budget = 20000
|
||||
compressor.context_length = 200000
|
||||
compressor.threshold_percent = 0.80
|
||||
compressor.threshold_tokens = 160000
|
||||
compressor.max_summary_tokens = 10000
|
||||
compressor.quiet_mode = True
|
||||
compressor.compression_count = 0
|
||||
compressor.last_prompt_tokens = 0
|
||||
compressor._previous_summary = None
|
||||
compressor._summary_failure_cooldown_until = 0.0
|
||||
compressor.summary_model = None
|
||||
compressor.model = "test-model"
|
||||
compressor.provider = "test"
|
||||
compressor.base_url = "http://localhost"
|
||||
compressor.api_key = "test-key"
|
||||
compressor.api_mode = "chat_completions"
|
||||
return compressor
|
||||
|
||||
|
||||
def test_focus_topic_injected_into_summary_prompt():
|
||||
"""When focus_topic is provided, the LLM prompt includes focus guidance."""
|
||||
compressor = _make_compressor()
|
||||
turns = [
|
||||
{"role": "user", "content": "Tell me about the database schema"},
|
||||
{"role": "assistant", "content": "The schema has tables: users, orders, products."},
|
||||
]
|
||||
|
||||
captured_prompt = {}
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
captured_prompt["messages"] = kwargs["messages"]
|
||||
resp = MagicMock()
|
||||
resp.choices = [MagicMock()]
|
||||
resp.choices[0].message.content = "## Goal\nUnderstand DB schema."
|
||||
return resp
|
||||
|
||||
with patch("agent.context_compressor.call_llm", mock_call_llm):
|
||||
result = compressor._generate_summary(turns, focus_topic="database schema")
|
||||
|
||||
assert result is not None
|
||||
prompt_text = captured_prompt["messages"][0]["content"]
|
||||
assert 'FOCUS TOPIC: "database schema"' in prompt_text
|
||||
assert "PRIORITISE" in prompt_text
|
||||
assert "60-70%" in prompt_text
|
||||
|
||||
|
||||
def test_no_focus_topic_no_injection():
|
||||
"""Without focus_topic, the prompt doesn't contain focus guidance."""
|
||||
compressor = _make_compressor()
|
||||
turns = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
]
|
||||
|
||||
captured_prompt = {}
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
captured_prompt["messages"] = kwargs["messages"]
|
||||
resp = MagicMock()
|
||||
resp.choices = [MagicMock()]
|
||||
resp.choices[0].message.content = "## Goal\nGreeting."
|
||||
return resp
|
||||
|
||||
with patch("agent.context_compressor.call_llm", mock_call_llm):
|
||||
result = compressor._generate_summary(turns)
|
||||
|
||||
prompt_text = captured_prompt["messages"][0]["content"]
|
||||
assert "FOCUS TOPIC" not in prompt_text
|
||||
|
||||
|
||||
def test_compress_passes_focus_to_generate_summary():
|
||||
"""compress() passes focus_topic through to _generate_summary."""
|
||||
compressor = _make_compressor()
|
||||
|
||||
# Track what _generate_summary receives
|
||||
received_kwargs = {}
|
||||
original_generate = compressor._generate_summary
|
||||
|
||||
def tracking_generate(turns, **kwargs):
|
||||
received_kwargs.update(kwargs)
|
||||
return "## Goal\nTest."
|
||||
|
||||
compressor._generate_summary = tracking_generate
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "System prompt"},
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "reply1"},
|
||||
{"role": "user", "content": "second"},
|
||||
{"role": "assistant", "content": "reply2"},
|
||||
{"role": "user", "content": "third"},
|
||||
{"role": "assistant", "content": "reply3"},
|
||||
{"role": "user", "content": "fourth"},
|
||||
{"role": "assistant", "content": "reply4"},
|
||||
]
|
||||
|
||||
compressor.compress(messages, current_tokens=100000, focus_topic="authentication flow")
|
||||
|
||||
assert received_kwargs.get("focus_topic") == "authentication flow"
|
||||
|
||||
|
||||
def test_compress_none_focus_by_default():
|
||||
"""compress() passes None focus_topic by default."""
|
||||
compressor = _make_compressor()
|
||||
|
||||
received_kwargs = {}
|
||||
|
||||
def tracking_generate(turns, **kwargs):
|
||||
received_kwargs.update(kwargs)
|
||||
return "## Goal\nTest."
|
||||
|
||||
compressor._generate_summary = tracking_generate
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "System prompt"},
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "reply1"},
|
||||
{"role": "user", "content": "second"},
|
||||
{"role": "assistant", "content": "reply2"},
|
||||
{"role": "user", "content": "third"},
|
||||
{"role": "assistant", "content": "reply3"},
|
||||
{"role": "user", "content": "fourth"},
|
||||
{"role": "assistant", "content": "reply4"},
|
||||
]
|
||||
|
||||
compressor.compress(messages, current_tokens=100000)
|
||||
|
||||
assert received_kwargs.get("focus_topic") is None
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Regression: prevent transcript fork when two paths compress the same session_id.
|
||||
|
||||
Damien's incident (Discord, 2026-05-28): a long Hermes session in a Discord
|
||||
gateway hit the compression threshold at the end of a turn. The parent agent
|
||||
finished delivering the response and ``conversation_loop.py`` fired
|
||||
``_spawn_background_review(...)`` — which builds a forked ``AIAgent`` that
|
||||
inherits ``agent.session_id`` (see ``agent/background_review.py``::
|
||||
``review_agent.session_id = agent.session_id``). Roughly two seconds later
|
||||
a synthetic ``Background process proc_… completed`` event arrived and
|
||||
started a fresh turn on the same parent ``session_id`` (still cached in the
|
||||
gateway's ``SessionEntry``). Both paths hit preflight compression on the
|
||||
same parent transcript and called ``_compress_context`` concurrently. Each
|
||||
ended the parent and created its own CHILD session in ``state.db``, both
|
||||
parented to the same old id. The gateway's ``SessionEntry`` only caught one
|
||||
rotation; the other child became an orphan that silently accumulated writes.
|
||||
|
||||
Repro shape on Damien's machine:
|
||||
|
||||
parent 20260527_234659_e65f0e ended_at=set end_reason='compression'
|
||||
child 20260528_113619_fc80e1 parent=20260527_234659_e65f0e (in SessionEntry)
|
||||
child <orphan> parent=20260527_234659_e65f0e (silent writes)
|
||||
|
||||
This regression simulates the two concurrent ``compress_context`` calls
|
||||
against a shared ``state.db`` and asserts that the per-session compression
|
||||
lock added in this PR prevents the orphan child. Without the lock the
|
||||
fixture deterministically produces 2 children; with the lock, exactly 1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB
|
||||
|
||||
|
||||
def _build_agent_with_db(db: SessionDB, session_id: str):
|
||||
"""Build an AIAgent that's wired to ``db`` and pinned to ``session_id``."""
|
||||
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
model="test/model",
|
||||
quiet_mode=True,
|
||||
session_db=db,
|
||||
session_id=session_id,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
|
||||
# Stub the compressor so it returns deterministic output and DOESN'T make
|
||||
# an LLM call. Sleep inside compress() so the two threads' rotations
|
||||
# actually overlap — without that the OS could happen to serialize them
|
||||
# and hide the bug.
|
||||
compressor = MagicMock()
|
||||
|
||||
def _compress_with_overlap(*_a, **_kw):
|
||||
time.sleep(0.25)
|
||||
return [
|
||||
{"role": "user", "content": "[CONTEXT COMPACTION] summary"},
|
||||
{"role": "user", "content": "tail"},
|
||||
]
|
||||
|
||||
compressor.compress.side_effect = _compress_with_overlap
|
||||
compressor.compression_count = 1
|
||||
compressor.last_prompt_tokens = 0
|
||||
compressor.last_completion_tokens = 0
|
||||
compressor._last_summary_error = None
|
||||
compressor._last_compress_aborted = False
|
||||
compressor._last_aux_model_failure_model = None
|
||||
compressor._last_aux_model_failure_error = None
|
||||
agent.context_compressor = compressor
|
||||
return agent
|
||||
|
||||
|
||||
def _count_children(db: SessionDB, parent_sid: str) -> int:
|
||||
"""Count rows in state.db whose parent_session_id == parent_sid."""
|
||||
rows = db._conn.execute(
|
||||
"SELECT id FROM sessions WHERE parent_session_id = ?",
|
||||
(parent_sid,),
|
||||
).fetchall()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def test_concurrent_compression_does_not_fork_session(tmp_path: Path) -> None:
|
||||
"""Two AIAgents that share a session_id MUST NOT both rotate it.
|
||||
|
||||
Without the per-session compression lock this fixture deterministically
|
||||
produces 2 child sessions (transcript fork). With the lock the second
|
||||
path aborts cleanly, leaving exactly 1 canonical child.
|
||||
"""
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
|
||||
parent_sid = "PARENT_TEST_SESSION"
|
||||
db.create_session(parent_sid, source="discord")
|
||||
|
||||
# Two agents on the same session_id, both wired to the same db —
|
||||
# mirrors the parent-turn agent + the background-review fork right
|
||||
# after a turn ends.
|
||||
agent_a = _build_agent_with_db(db, parent_sid)
|
||||
agent_b = _build_agent_with_db(db, parent_sid)
|
||||
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
|
||||
|
||||
def run(agent):
|
||||
try:
|
||||
agent._compress_context(messages, "sys", approx_tokens=120_000)
|
||||
except Exception:
|
||||
# Surface to the test if either raises — should not happen.
|
||||
raise
|
||||
|
||||
t_a = threading.Thread(target=run, args=(agent_a,), name="main_turn")
|
||||
t_b = threading.Thread(target=run, args=(agent_b,), name="review_fork")
|
||||
t_a.start()
|
||||
t_b.start()
|
||||
t_a.join(timeout=10)
|
||||
t_b.join(timeout=10)
|
||||
|
||||
# Exactly one canonical child — not two orphans.
|
||||
assert _count_children(db, parent_sid) == 1, (
|
||||
"Compression lock failed: parent session has multiple children in state.db "
|
||||
"(transcript fork). This is Damien's incident shape — see the test docstring."
|
||||
)
|
||||
|
||||
# And exactly one of the two agents actually rotated its session_id; the
|
||||
# other should still hold the parent_sid (its compression was skipped).
|
||||
rotated = sum(
|
||||
1 for a in (agent_a, agent_b) if a.session_id != parent_sid
|
||||
)
|
||||
assert rotated == 1, (
|
||||
f"Expected exactly one agent to rotate session_id, got {rotated}. "
|
||||
"Both agents rotating means the lock didn't serialize them."
|
||||
)
|
||||
|
||||
# The lock must be released after the winner finished.
|
||||
assert db.get_compression_lock_holder(parent_sid) is None, (
|
||||
"Compression lock leaked: still held after both rotations completed."
|
||||
)
|
||||
|
||||
|
||||
def test_skipped_compression_returns_messages_unchanged(tmp_path: Path) -> None:
|
||||
"""The loser of the lock race must return its input messages verbatim.
|
||||
|
||||
Callers (preflight compression in ``conversation_loop.py``) detect the
|
||||
no-op via ``len(returned) == len(input)`` and stop the auto-compress
|
||||
retry loop. If the skipped path returned the compressed view, that
|
||||
detection would break and the caller would mutate the conversation
|
||||
without going through state.db rotation.
|
||||
"""
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
parent_sid = "LOSER_TEST"
|
||||
db.create_session(parent_sid, source="discord")
|
||||
|
||||
# Pre-acquire the lock so the agent's compress_context sees it held.
|
||||
held = db.try_acquire_compression_lock(parent_sid, "external_holder")
|
||||
assert held is True
|
||||
|
||||
agent = _build_agent_with_db(db, parent_sid)
|
||||
messages = [{"role": "user", "content": "m1"}, {"role": "user", "content": "m2"}]
|
||||
|
||||
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
|
||||
|
||||
# Skipped: messages returned verbatim, no rotation
|
||||
assert compressed is messages or compressed == messages
|
||||
assert agent.session_id == parent_sid
|
||||
# Compressor was never called (the skip happens before .compress())
|
||||
agent.context_compressor.compress.assert_not_called()
|
||||
|
||||
|
||||
class _NoLockSubsystemDB:
|
||||
"""Wraps a real SessionDB but simulates a pre-#34351 version skew.
|
||||
|
||||
A long-lived process can hold ``hermes_state.SessionDB`` bound to the
|
||||
OLD class in memory (no compression-lock methods) while a lazily
|
||||
re-imported ``conversation_compression.py`` calls the NEW lock code.
|
||||
``try_acquire_compression_lock`` then raises ``AttributeError`` — which
|
||||
is NOT a ``sqlite3.Error``, so the method's own fail-open guard never
|
||||
runs. Before the fix the exception propagated to the outer agent loop,
|
||||
which printed the error and retried; compression never succeeded, the
|
||||
token count never dropped, and the loop re-triggered compaction forever.
|
||||
"""
|
||||
|
||||
def __init__(self, real_db: SessionDB) -> None:
|
||||
self._real = real_db
|
||||
|
||||
def try_acquire_compression_lock(self, *_a, **_k): # noqa: D401
|
||||
raise AttributeError(
|
||||
"'SessionDB' object has no attribute 'try_acquire_compression_lock'"
|
||||
)
|
||||
|
||||
def get_compression_lock_holder(self, *_a, **_k):
|
||||
raise AttributeError("'SessionDB' object has no attribute 'get_compression_lock_holder'")
|
||||
|
||||
def release_compression_lock(self, *_a, **_k):
|
||||
raise AttributeError("'SessionDB' object has no attribute 'release_compression_lock'")
|
||||
|
||||
def __getattr__(self, name):
|
||||
# Everything else (create_session, append, rotation helpers) goes to
|
||||
# the real db so the post-lock compression + rotation path runs.
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
def test_missing_lock_subsystem_fails_open_not_infinite_loop(tmp_path: Path) -> None:
|
||||
"""Version skew (no lock methods) must fail OPEN, not raise into the loop.
|
||||
|
||||
Reproduces the "API call #47/#48/#49 ... has no attribute
|
||||
try_acquire_compression_lock" infinite-compaction spin: when the lock
|
||||
subsystem is absent, ``_compress_context`` must skip locking and proceed
|
||||
with compression (so the loop makes progress and terminates) instead of
|
||||
letting the ``AttributeError`` escape to the retry loop.
|
||||
"""
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
parent_sid = "SKEW_TEST_SESSION"
|
||||
db.create_session(parent_sid, source="discord")
|
||||
|
||||
agent = _build_agent_with_db(db, parent_sid)
|
||||
# Swap in the lock-less wrapper AFTER construction (the agent already
|
||||
# holds a normal db reference; we only break the lock methods).
|
||||
agent._session_db = _NoLockSubsystemDB(db)
|
||||
|
||||
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
|
||||
|
||||
# MUST NOT raise AttributeError. Before the fix this raised and the
|
||||
# outer loop would retry forever.
|
||||
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
|
||||
|
||||
# Compression actually ran (proceeded past the broken lock) and made
|
||||
# progress, so the auto-compress loop would terminate.
|
||||
agent.context_compressor.compress.assert_called_once()
|
||||
assert len(compressed) < len(messages), (
|
||||
"Compression made no progress despite failing open — loop would still spin."
|
||||
)
|
||||
# Session rotated (compression succeeded end-to-end).
|
||||
assert agent.session_id != parent_sid
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Tests for post-compression historical-media stripping.
|
||||
|
||||
Port of Kilo-Org/kilocode#9434 (adapted for OpenAI-style message lists).
|
||||
Without this pass, tail messages keep their original multi-MB base-64 image
|
||||
payloads after context compression, and every subsequent request re-ships
|
||||
them — sometimes breaching provider body-size limits and wedging the
|
||||
session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.context_compressor import (
|
||||
ContextCompressor,
|
||||
_content_has_images,
|
||||
_is_image_part,
|
||||
_strip_historical_media,
|
||||
_strip_images_from_content,
|
||||
)
|
||||
|
||||
|
||||
IMG_URL = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64," + ("A" * 1024)},
|
||||
}
|
||||
INPUT_IMG = {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64," + ("B" * 1024),
|
||||
}
|
||||
ANTHROPIC_IMG = {
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": "C" * 1024},
|
||||
}
|
||||
TEXT = {"type": "text", "text": "hi"}
|
||||
INPUT_TEXT = {"type": "input_text", "text": "hi"}
|
||||
|
||||
|
||||
class TestIsImagePart:
|
||||
def test_openai_chat_shape(self):
|
||||
assert _is_image_part(IMG_URL) is True
|
||||
|
||||
def test_openai_responses_shape(self):
|
||||
assert _is_image_part(INPUT_IMG) is True
|
||||
|
||||
def test_anthropic_native_shape(self):
|
||||
assert _is_image_part(ANTHROPIC_IMG) is True
|
||||
|
||||
def test_text_part_is_not_image(self):
|
||||
assert _is_image_part(TEXT) is False
|
||||
assert _is_image_part(INPUT_TEXT) is False
|
||||
|
||||
def test_non_dict_rejected(self):
|
||||
assert _is_image_part("image") is False
|
||||
assert _is_image_part(None) is False
|
||||
assert _is_image_part(42) is False
|
||||
|
||||
|
||||
class TestContentHasImages:
|
||||
def test_string_content(self):
|
||||
assert _content_has_images("a string") is False
|
||||
|
||||
def test_empty_list(self):
|
||||
assert _content_has_images([]) is False
|
||||
|
||||
def test_text_only_list(self):
|
||||
assert _content_has_images([TEXT, TEXT]) is False
|
||||
|
||||
def test_list_with_image(self):
|
||||
assert _content_has_images([TEXT, IMG_URL]) is True
|
||||
|
||||
def test_none(self):
|
||||
assert _content_has_images(None) is False
|
||||
|
||||
|
||||
class TestStripImagesFromContent:
|
||||
def test_string_passthrough(self):
|
||||
assert _strip_images_from_content("hello") == "hello"
|
||||
|
||||
def test_none_passthrough(self):
|
||||
assert _strip_images_from_content(None) is None
|
||||
|
||||
def test_text_only_passthrough(self):
|
||||
parts = [TEXT, {"type": "text", "text": "world"}]
|
||||
assert _strip_images_from_content(parts) == parts
|
||||
|
||||
def test_replaces_image_with_placeholder(self):
|
||||
parts = [TEXT, IMG_URL]
|
||||
out = _strip_images_from_content(parts)
|
||||
assert len(out) == 2
|
||||
assert out[0] == TEXT
|
||||
assert out[1] == {
|
||||
"type": "text",
|
||||
"text": "[Attached image — stripped after compression]",
|
||||
}
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
parts = [IMG_URL, TEXT]
|
||||
_ = _strip_images_from_content(parts)
|
||||
assert parts[0] is IMG_URL # original list untouched
|
||||
assert parts[1] is TEXT
|
||||
|
||||
def test_handles_all_three_shapes(self):
|
||||
parts = [IMG_URL, INPUT_IMG, ANTHROPIC_IMG, TEXT]
|
||||
out = _strip_images_from_content(parts)
|
||||
assert sum(1 for p in out if p.get("type") == "text") == 4
|
||||
assert not any(_is_image_part(p) for p in out)
|
||||
|
||||
|
||||
class TestStripHistoricalMedia:
|
||||
def test_empty_passthrough(self):
|
||||
assert _strip_historical_media([]) == []
|
||||
|
||||
def test_no_images_anywhere(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hey"},
|
||||
{"role": "user", "content": "bye"},
|
||||
]
|
||||
assert _strip_historical_media(msgs) is msgs # identity — no copy
|
||||
|
||||
def test_single_image_user_only_first_message(self):
|
||||
# Only image-bearing user is the first message — nothing before it.
|
||||
msgs = [
|
||||
{"role": "user", "content": [TEXT, IMG_URL]},
|
||||
{"role": "assistant", "content": "ok"},
|
||||
]
|
||||
out = _strip_historical_media(msgs)
|
||||
assert out is msgs # no-op
|
||||
# Image still there.
|
||||
assert _content_has_images(out[0]["content"])
|
||||
|
||||
def test_strips_older_user_image_keeps_newest(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": [TEXT, IMG_URL]}, # old — strip
|
||||
{"role": "assistant", "content": "looked at it"},
|
||||
{"role": "user", "content": [TEXT, INPUT_IMG]}, # newest — keep
|
||||
]
|
||||
out = _strip_historical_media(msgs)
|
||||
assert out is not msgs # new list
|
||||
# First message's image was replaced
|
||||
assert not _content_has_images(out[0]["content"])
|
||||
# Newest user still has its image
|
||||
assert _content_has_images(out[2]["content"])
|
||||
|
||||
def test_strips_assistant_and_tool_images_before_anchor(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": [TEXT, IMG_URL]}, # old user
|
||||
{"role": "assistant", "content": [TEXT, IMG_URL]}, # old assistant
|
||||
{"role": "tool", "content": [TEXT, IMG_URL], "tool_call_id": "t1"},
|
||||
{"role": "user", "content": [TEXT, IMG_URL]}, # newest user — keep
|
||||
]
|
||||
out = _strip_historical_media(msgs)
|
||||
for i in range(3):
|
||||
assert not _content_has_images(out[i]["content"]), f"msg {i} still has image"
|
||||
assert _content_has_images(out[3]["content"])
|
||||
|
||||
def test_text_only_newest_user_still_strips_older_images(self):
|
||||
# The anchor is "newest user WITH images". If the newest user is
|
||||
# text-only, we fall back to the previous image-bearing user turn.
|
||||
msgs = [
|
||||
{"role": "user", "content": [TEXT, IMG_URL]},
|
||||
{"role": "assistant", "content": "ok"},
|
||||
{"role": "user", "content": [TEXT, IMG_URL]}, # anchor
|
||||
{"role": "assistant", "content": "done"},
|
||||
{"role": "user", "content": "follow-up text only"},
|
||||
]
|
||||
out = _strip_historical_media(msgs)
|
||||
# First image-bearing user (index 0) was stripped — it was before the
|
||||
# newest image-bearing user (index 2).
|
||||
assert not _content_has_images(out[0]["content"])
|
||||
# Anchor (index 2) keeps its image.
|
||||
assert _content_has_images(out[2]["content"])
|
||||
|
||||
def test_no_image_bearing_user_is_noop(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": [TEXT, IMG_URL]}, # assistant image only
|
||||
{"role": "user", "content": "second"},
|
||||
]
|
||||
out = _strip_historical_media(msgs)
|
||||
# No image-bearing user anchor → no stripping.
|
||||
assert out is msgs
|
||||
assert _content_has_images(out[1]["content"])
|
||||
|
||||
def test_does_not_mutate_input_messages(self):
|
||||
msg0 = {"role": "user", "content": [TEXT, IMG_URL]}
|
||||
msg1 = {"role": "user", "content": [TEXT, IMG_URL]}
|
||||
msgs = [msg0, msg1]
|
||||
_ = _strip_historical_media(msgs)
|
||||
# Originals untouched
|
||||
assert _content_has_images(msg0["content"])
|
||||
assert _content_has_images(msg1["content"])
|
||||
|
||||
def test_idempotent(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": [TEXT, IMG_URL]},
|
||||
{"role": "assistant", "content": "k"},
|
||||
{"role": "user", "content": [TEXT, IMG_URL]},
|
||||
]
|
||||
first = _strip_historical_media(msgs)
|
||||
second = _strip_historical_media(first)
|
||||
# Second pass is a no-op — no images left before the anchor.
|
||||
assert second is first
|
||||
|
||||
def test_non_dict_messages_pass_through(self):
|
||||
msgs = [
|
||||
"not-a-dict", # shouldn't crash
|
||||
{"role": "user", "content": [TEXT, IMG_URL]},
|
||||
{"role": "assistant", "content": "ok"},
|
||||
{"role": "user", "content": [TEXT, IMG_URL]},
|
||||
]
|
||||
out = _strip_historical_media(msgs)
|
||||
assert out[0] == "not-a-dict"
|
||||
# Image-bearing user at index 1 is before the anchor (index 3) → stripped.
|
||||
assert not _content_has_images(out[1]["content"])
|
||||
|
||||
|
||||
class TestCompressIntegration:
|
||||
"""Verify the stripping runs inside ContextCompressor.compress()."""
|
||||
|
||||
@pytest.fixture
|
||||
def compressor(self):
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(
|
||||
model="test/model",
|
||||
threshold_percent=0.50,
|
||||
protect_first_n=1,
|
||||
protect_last_n=2,
|
||||
quiet_mode=True,
|
||||
)
|
||||
return c
|
||||
|
||||
def test_compress_strips_historical_images(self, compressor):
|
||||
# Enough messages to trigger the summarize path. protect_first_n=1 +
|
||||
# protect_last_n=2 + a middle window of at least 3 with a summary.
|
||||
msgs = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": [TEXT, IMG_URL]}, # old image-bearing user
|
||||
{"role": "assistant", "content": "looked at it"},
|
||||
{"role": "user", "content": "follow-up"},
|
||||
{"role": "assistant", "content": "ack"},
|
||||
{"role": "user", "content": "more"},
|
||||
{"role": "assistant", "content": "ok"},
|
||||
{"role": "user", "content": [TEXT, IMG_URL]}, # newest image-bearing user (tail)
|
||||
{"role": "assistant", "content": "done"},
|
||||
]
|
||||
# Bypass the real LLM summary — return a stub so compress() proceeds.
|
||||
with patch.object(compressor, "_generate_summary", return_value="SUMMARY TEXT"):
|
||||
out = compressor.compress(msgs, current_tokens=60_000)
|
||||
|
||||
# Newest user turn with image should still have it (it's in the tail).
|
||||
user_imgs = [m for m in out if m.get("role") == "user" and _content_has_images(m.get("content"))]
|
||||
assert len(user_imgs) == 1, (
|
||||
"Expected exactly one user message with images after compression "
|
||||
f"(the newest one); got {len(user_imgs)}"
|
||||
)
|
||||
# No assistant or tool messages should carry images either.
|
||||
for m in out:
|
||||
if m is user_imgs[0]:
|
||||
continue
|
||||
assert not _content_has_images(m.get("content")), (
|
||||
f"Stale image in {m.get('role')!r} message after compression"
|
||||
)
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Tests for image-token accounting in the context compressor.
|
||||
|
||||
Covers the native-image-routing PR's companion change: the compressor's
|
||||
multimodal message length counter now charges ~1600 tokens per attached
|
||||
image part instead of 0, so tail-cut / prune decisions are accurate for
|
||||
creative workflows that iterate on images across many turns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from agent.context_compressor import (
|
||||
_CHARS_PER_TOKEN,
|
||||
_IMAGE_CHAR_EQUIVALENT,
|
||||
_IMAGE_TOKEN_ESTIMATE,
|
||||
_content_length_for_budget,
|
||||
)
|
||||
|
||||
|
||||
class TestContentLengthForBudget:
|
||||
def test_plain_string(self):
|
||||
assert _content_length_for_budget("hello world") == 11
|
||||
|
||||
def test_empty_string(self):
|
||||
assert _content_length_for_budget("") == 0
|
||||
|
||||
def test_none_coerces_to_zero(self):
|
||||
assert _content_length_for_budget(None) == 0
|
||||
|
||||
def test_text_only_list(self):
|
||||
content = [
|
||||
{"type": "text", "text": "first"},
|
||||
{"type": "text", "text": "second"},
|
||||
]
|
||||
assert _content_length_for_budget(content) == 5 + 6
|
||||
|
||||
def test_single_image_part_charges_fixed_budget(self):
|
||||
content = [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,XXXX"}},
|
||||
]
|
||||
# 4 chars of text + 1 image at fixed char-equivalent
|
||||
assert _content_length_for_budget(content) == 4 + _IMAGE_CHAR_EQUIVALENT
|
||||
|
||||
def test_image_url_raw_base64_is_not_counted_as_chars(self):
|
||||
"""A 1MB base64 blob inside an image_url must NOT inflate token count.
|
||||
|
||||
The flat image estimate is what the provider actually bills; the raw
|
||||
base64 is transport payload, not context tokens.
|
||||
"""
|
||||
huge_url = "data:image/png;base64," + ("A" * 1_000_000)
|
||||
content = [
|
||||
{"type": "image_url", "image_url": {"url": huge_url}},
|
||||
]
|
||||
# Exactly one image's worth, not 1M + something.
|
||||
assert _content_length_for_budget(content) == _IMAGE_CHAR_EQUIVALENT
|
||||
|
||||
def test_multiple_image_parts(self):
|
||||
content = [
|
||||
{"type": "text", "text": "compare"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,BBB"}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,CCC"}},
|
||||
]
|
||||
assert _content_length_for_budget(content) == 7 + 3 * _IMAGE_CHAR_EQUIVALENT
|
||||
|
||||
def test_openai_responses_input_image_shape(self):
|
||||
"""Responses API uses type=input_image with top-level image_url string."""
|
||||
content = [
|
||||
{"type": "input_text", "text": "hey"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,XX"},
|
||||
]
|
||||
# input_text has .text "hey" (3 chars) + 1 image
|
||||
assert _content_length_for_budget(content) == 3 + _IMAGE_CHAR_EQUIVALENT
|
||||
|
||||
def test_anthropic_native_image_shape(self):
|
||||
"""Anthropic native shape: {type: image, source: {...}}."""
|
||||
content = [
|
||||
{"type": "text", "text": "hi"},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "XX"}},
|
||||
]
|
||||
assert _content_length_for_budget(content) == 2 + _IMAGE_CHAR_EQUIVALENT
|
||||
|
||||
def test_bare_string_part_in_list(self):
|
||||
"""Older code paths sometimes produce mixed list-of-strings content."""
|
||||
content = ["hello", {"type": "text", "text": "world"}]
|
||||
assert _content_length_for_budget(content) == 5 + 5
|
||||
|
||||
def test_image_estimate_constant_is_reasonable(self):
|
||||
"""Sanity-check the estimate aligns with real provider billing.
|
||||
|
||||
Anthropic ≈ width*height/750 → ~1600 for 1000×1200.
|
||||
OpenAI GPT-4o high-detail 2048×2048 ≈ 1445.
|
||||
Gemini 258/tile × 6 tiles for a 2048×2048 ≈ 1548.
|
||||
Anything in the 800-2000 range is defensible. Enforce bounds so an
|
||||
accidental edit doesn't drop it to e.g. 16.
|
||||
"""
|
||||
assert 800 <= _IMAGE_TOKEN_ESTIMATE <= 2500
|
||||
assert _IMAGE_CHAR_EQUIVALENT == _IMAGE_TOKEN_ESTIMATE * _CHARS_PER_TOKEN
|
||||
|
||||
|
||||
class TestTokenBudgetWithImages:
|
||||
"""Integration: the compressor's tail-cut decision now respects image cost."""
|
||||
|
||||
def test_image_heavy_turns_count_toward_budget(self):
|
||||
"""A tail with 5 image-bearing turns should blow past a 5K token budget."""
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
# Minimal compressor fixture — just enough to call _find_tail_cut_by_tokens
|
||||
cc = object.__new__(ContextCompressor)
|
||||
cc.tail_token_budget = 5000
|
||||
|
||||
# Build 10 messages: 5 with images, 5 with short text. Without the
|
||||
# image-tokens fix, the compressor would think all 10 fit in 5K and
|
||||
# protect them all. With the fix, images alone cost 5 × 1600 = 8K,
|
||||
# so the tail should be trimmed.
|
||||
messages = [{"role": "system", "content": "sys"}]
|
||||
for i in range(5):
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": f"turn {i}"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},
|
||||
],
|
||||
})
|
||||
messages.append({
|
||||
"role": "assistant",
|
||||
"content": f"response {i}",
|
||||
})
|
||||
|
||||
cut = cc._find_tail_cut_by_tokens(messages, head_end=0, token_budget=5000)
|
||||
|
||||
# Budget is 5K, soft ceiling 7.5K. 5 images alone = 8000 image-tokens.
|
||||
# Walking backward, the compressor should stop before including all 5.
|
||||
# Exact cut depends on text lengths and min_tail, but it MUST be > 1
|
||||
# (at least some head-side messages should be compressible).
|
||||
assert cut > 1, (
|
||||
f"Expected image-heavy tail to be trimmed; compressor placed cut at "
|
||||
f"{cut} out of {len(messages)} (image tokens were likely ignored)."
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,87 @@
|
||||
"""Regression tests for iterative context-summary continuity."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from agent.context_compressor import ContextCompressor, SUMMARY_PREFIX
|
||||
|
||||
|
||||
def _compressor() -> ContextCompressor:
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
return ContextCompressor(
|
||||
model="test/model",
|
||||
threshold_percent=0.85,
|
||||
protect_first_n=1,
|
||||
protect_last_n=1,
|
||||
quiet_mode=True,
|
||||
)
|
||||
|
||||
|
||||
def _response(content: str):
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = content
|
||||
return mock_response
|
||||
|
||||
|
||||
def _messages_with_handoff(summary_body: str):
|
||||
return [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{summary_body}"},
|
||||
{"role": "assistant", "content": "handoff acknowledged after resume"},
|
||||
{"role": "user", "content": "new user turn after resume"},
|
||||
{"role": "assistant", "content": "new assistant work after resume"},
|
||||
{"role": "user", "content": "more new work after resume"},
|
||||
{"role": "assistant", "content": "latest tail response"},
|
||||
{"role": "user", "content": "final active request stays in protected tail"},
|
||||
]
|
||||
|
||||
|
||||
def test_existing_previous_summary_is_not_serialized_again_as_new_turn():
|
||||
"""Same-process iterative compression should not feed the old handoff twice."""
|
||||
compressor = _compressor()
|
||||
old_summary = "OLD-SUMMARY-BODY unique continuity facts"
|
||||
compressor._previous_summary = old_summary
|
||||
|
||||
with patch("agent.context_compressor.call_llm", return_value=_response("updated summary")) as mock_call:
|
||||
compressor.compress(_messages_with_handoff(old_summary))
|
||||
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
assert "PREVIOUS SUMMARY:" in prompt
|
||||
assert "NEW TURNS TO INCORPORATE:" in prompt
|
||||
assert prompt.count(old_summary) == 1
|
||||
assert f"[USER]: {SUMMARY_PREFIX}" not in prompt
|
||||
|
||||
|
||||
def test_resume_rehydrates_previous_summary_from_handoff_message():
|
||||
"""After restart/resume, the persisted handoff should regain summary identity."""
|
||||
compressor = _compressor()
|
||||
old_summary = "RESUMED-SUMMARY-BODY durable continuity facts"
|
||||
assert compressor._previous_summary is None
|
||||
|
||||
with patch("agent.context_compressor.call_llm", return_value=_response("updated summary")) as mock_call:
|
||||
compressor.compress(_messages_with_handoff(old_summary))
|
||||
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
assert "PREVIOUS SUMMARY:" in prompt
|
||||
assert "NEW TURNS TO INCORPORATE:" in prompt
|
||||
assert "TURNS TO SUMMARIZE:" not in prompt
|
||||
assert prompt.count(old_summary) == 1
|
||||
assert f"[USER]: {SUMMARY_PREFIX}" not in prompt
|
||||
|
||||
|
||||
def test_handoff_in_protected_head_populates_previous_summary_before_update():
|
||||
"""A resumed protected-head handoff should restore iterative-summary state."""
|
||||
compressor = _compressor()
|
||||
old_summary = "PROTECTED-HEAD-SUMMARY durable facts from before restart"
|
||||
seen_turns = []
|
||||
|
||||
def fake_generate_summary(turns_to_summarize, focus_topic=None):
|
||||
seen_turns.extend(turns_to_summarize)
|
||||
return "new summary from resumed turns"
|
||||
|
||||
with patch.object(compressor, "_generate_summary", side_effect=fake_generate_summary):
|
||||
compressor.compress(_messages_with_handoff(old_summary))
|
||||
|
||||
assert compressor._previous_summary == old_summary
|
||||
assert seen_turns
|
||||
assert all(old_summary not in str(msg.get("content", "")) for msg in seen_turns)
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Tests for the ContextEngine ABC and plugin slot."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from agent.context_engine import ContextEngine
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# A minimal concrete engine for testing the ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class StubEngine(ContextEngine):
|
||||
"""Minimal engine that satisfies the ABC without doing real work."""
|
||||
|
||||
def __init__(self, context_length=200000, threshold_pct=0.50):
|
||||
self.context_length = context_length
|
||||
self.threshold_tokens = int(context_length * threshold_pct)
|
||||
self._compress_called = False
|
||||
self._tools_called = []
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "stub"
|
||||
|
||||
def update_from_response(self, usage: Dict[str, Any]) -> None:
|
||||
self.last_prompt_tokens = usage.get("prompt_tokens", 0)
|
||||
self.last_completion_tokens = usage.get("completion_tokens", 0)
|
||||
self.last_total_tokens = usage.get("total_tokens", 0)
|
||||
|
||||
def should_compress(self, prompt_tokens: int = None) -> bool:
|
||||
tokens = prompt_tokens if prompt_tokens is not None else self.last_prompt_tokens
|
||||
return tokens >= self.threshold_tokens
|
||||
|
||||
def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) -> List[Dict[str, Any]]:
|
||||
self._compress_called = True
|
||||
self.compression_count += 1
|
||||
# Trivial: just return as-is
|
||||
return messages
|
||||
|
||||
def get_tool_schemas(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"name": "stub_search",
|
||||
"description": "Search the stub engine",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
]
|
||||
|
||||
def handle_tool_call(self, name: str, args: Dict[str, Any]) -> str:
|
||||
self._tools_called.append(name)
|
||||
return json.dumps({"ok": True, "tool": name})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ABC contract tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestContextEngineABC:
|
||||
"""Verify the ABC enforces the required interface."""
|
||||
|
||||
def test_cannot_instantiate_abc_directly(self):
|
||||
with pytest.raises(TypeError):
|
||||
ContextEngine()
|
||||
|
||||
def test_missing_methods_raises(self):
|
||||
"""A subclass missing required methods cannot be instantiated."""
|
||||
class Incomplete(ContextEngine):
|
||||
@property
|
||||
def name(self):
|
||||
return "incomplete"
|
||||
with pytest.raises(TypeError):
|
||||
Incomplete()
|
||||
|
||||
def test_stub_engine_satisfies_abc(self):
|
||||
engine = StubEngine()
|
||||
assert isinstance(engine, ContextEngine)
|
||||
assert engine.name == "stub"
|
||||
|
||||
def test_compressor_is_context_engine(self):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
|
||||
assert isinstance(c, ContextEngine)
|
||||
assert c.name == "compressor"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default method behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDefaults:
|
||||
"""Verify ABC default implementations work correctly."""
|
||||
|
||||
def test_default_tool_schemas_empty(self):
|
||||
engine = StubEngine()
|
||||
# StubEngine overrides this, so test the base via super
|
||||
assert ContextEngine.get_tool_schemas(engine) == []
|
||||
|
||||
def test_default_handle_tool_call_returns_error(self):
|
||||
engine = StubEngine()
|
||||
result = ContextEngine.handle_tool_call(engine, "unknown", {})
|
||||
data = json.loads(result)
|
||||
assert "error" in data
|
||||
|
||||
def test_default_get_status(self):
|
||||
engine = StubEngine()
|
||||
engine.last_prompt_tokens = 50000
|
||||
status = engine.get_status()
|
||||
assert status["last_prompt_tokens"] == 50000
|
||||
assert status["context_length"] == 200000
|
||||
assert status["threshold_tokens"] == 100000
|
||||
assert 0 < status["usage_percent"] <= 100
|
||||
|
||||
def test_on_session_reset(self):
|
||||
engine = StubEngine()
|
||||
engine.last_prompt_tokens = 999
|
||||
engine.compression_count = 3
|
||||
engine.on_session_reset()
|
||||
assert engine.last_prompt_tokens == 0
|
||||
assert engine.compression_count == 0
|
||||
|
||||
def test_should_compress_preflight_default_false(self):
|
||||
engine = StubEngine()
|
||||
assert engine.should_compress_preflight([]) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StubEngine behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestStubEngine:
|
||||
|
||||
def test_should_compress(self):
|
||||
engine = StubEngine(context_length=100000, threshold_pct=0.50)
|
||||
assert not engine.should_compress(40000)
|
||||
assert engine.should_compress(50000)
|
||||
assert engine.should_compress(60000)
|
||||
|
||||
def test_compress_tracks_count(self):
|
||||
engine = StubEngine()
|
||||
msgs = [{"role": "user", "content": "hello"}]
|
||||
result = engine.compress(msgs)
|
||||
assert result == msgs
|
||||
assert engine._compress_called
|
||||
assert engine.compression_count == 1
|
||||
|
||||
def test_tool_schemas(self):
|
||||
engine = StubEngine()
|
||||
schemas = engine.get_tool_schemas()
|
||||
assert len(schemas) == 1
|
||||
assert schemas[0]["name"] == "stub_search"
|
||||
|
||||
def test_handle_tool_call(self):
|
||||
engine = StubEngine()
|
||||
result = engine.handle_tool_call("stub_search", {})
|
||||
assert json.loads(result)["ok"] is True
|
||||
assert "stub_search" in engine._tools_called
|
||||
|
||||
def test_update_from_response(self):
|
||||
engine = StubEngine()
|
||||
engine.update_from_response({"prompt_tokens": 1000, "completion_tokens": 200, "total_tokens": 1200})
|
||||
assert engine.last_prompt_tokens == 1000
|
||||
assert engine.last_completion_tokens == 200
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextCompressor session reset via ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCompressorSessionReset:
|
||||
"""Verify ContextCompressor.on_session_reset() clears all state."""
|
||||
|
||||
def test_reset_clears_state(self):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
|
||||
c.last_prompt_tokens = 50000
|
||||
c.compression_count = 3
|
||||
c._previous_summary = "some old summary"
|
||||
c._context_probed = True
|
||||
c._context_probe_persistable = True
|
||||
|
||||
c.on_session_reset()
|
||||
|
||||
assert c.last_prompt_tokens == 0
|
||||
assert c.last_completion_tokens == 0
|
||||
assert c.last_total_tokens == 0
|
||||
assert c.compression_count == 0
|
||||
assert c._context_probed is False
|
||||
assert c._context_probe_persistable is False
|
||||
assert c._previous_summary is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin slot (PluginManager integration)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPluginContextEngineSlot:
|
||||
"""Test register_context_engine on PluginContext."""
|
||||
|
||||
def test_register_engine(self):
|
||||
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest
|
||||
mgr = PluginManager()
|
||||
manifest = PluginManifest(name="test-lcm")
|
||||
ctx = PluginContext(manifest, mgr)
|
||||
|
||||
engine = StubEngine()
|
||||
ctx.register_context_engine(engine)
|
||||
|
||||
assert mgr._context_engine is engine
|
||||
assert mgr._context_engine.name == "stub"
|
||||
|
||||
def test_reject_second_engine(self):
|
||||
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest
|
||||
mgr = PluginManager()
|
||||
manifest = PluginManifest(name="test-lcm")
|
||||
ctx = PluginContext(manifest, mgr)
|
||||
|
||||
engine1 = StubEngine()
|
||||
engine2 = StubEngine()
|
||||
ctx.register_context_engine(engine1)
|
||||
ctx.register_context_engine(engine2) # should be rejected
|
||||
|
||||
assert mgr._context_engine is engine1
|
||||
|
||||
def test_reject_non_engine(self):
|
||||
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest
|
||||
mgr = PluginManager()
|
||||
manifest = PluginManifest(name="test-bad")
|
||||
ctx = PluginContext(manifest, mgr)
|
||||
|
||||
ctx.register_context_engine("not an engine")
|
||||
assert mgr._context_engine is None
|
||||
|
||||
def test_get_plugin_context_engine(self):
|
||||
from hermes_cli.plugins import PluginManager, get_plugin_context_engine
|
||||
import hermes_cli.plugins as plugins_mod
|
||||
|
||||
# Inject a test manager
|
||||
old_mgr = plugins_mod._plugin_manager
|
||||
try:
|
||||
mgr = PluginManager()
|
||||
plugins_mod._plugin_manager = mgr
|
||||
|
||||
assert get_plugin_context_engine() is None
|
||||
|
||||
engine = StubEngine()
|
||||
mgr._context_engine = engine
|
||||
assert get_plugin_context_engine() is engine
|
||||
finally:
|
||||
plugins_mod._plugin_manager = old_mgr
|
||||
@@ -0,0 +1,289 @@
|
||||
"""Regressions for the context-engine host contract.
|
||||
|
||||
These tests pin the five generic host-side guarantees that external context
|
||||
engine plugins (e.g. hermes-lcm) rely on:
|
||||
|
||||
1. ``_transition_context_engine_session`` drives the full lifecycle
|
||||
(on_session_end → on_session_reset → on_session_start → optional
|
||||
carry_over_new_session_context) and ``reset_session_state`` delegates
|
||||
to it when callers pass session metadata.
|
||||
|
||||
2. ``on_session_start`` receives ``conversation_id`` derived from
|
||||
``_gateway_session_key`` at agent init time.
|
||||
|
||||
3. ``conversation_loop`` forwards canonical cache buckets
|
||||
(``cache_read_tokens``, ``cache_write_tokens``, ``input_tokens``,
|
||||
``output_tokens``, ``reasoning_tokens``) to the engine's
|
||||
``update_from_response``, on top of the legacy aggregate keys.
|
||||
|
||||
4. ``_discover_context_engines`` includes plugin-registered engines (not
|
||||
just repo-shipped engines under ``plugins/context_engine/``).
|
||||
|
||||
5. The repo-shipped ``_EngineCollector`` honors ``ctx.register_command``
|
||||
from a plugin engine's ``register(ctx)`` entry point and routes it
|
||||
to the global plugin command registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
||||
from run_agent import AIAgent
|
||||
|
||||
|
||||
def _bare_agent() -> AIAgent:
|
||||
agent = object.__new__(AIAgent)
|
||||
agent.session_id = "test-session"
|
||||
agent.model = "fake-model"
|
||||
agent.platform = "telegram"
|
||||
agent._gateway_session_key = "agent:main:telegram:dm:42"
|
||||
return agent
|
||||
|
||||
|
||||
def test_transition_runs_full_lifecycle_in_order():
|
||||
"""End → reset → start → carry_over, in that order, when all inputs apply."""
|
||||
events: list[str] = []
|
||||
engine = MagicMock()
|
||||
engine.context_length = 200_000
|
||||
engine.on_session_end.side_effect = lambda *a, **kw: events.append("on_session_end")
|
||||
engine.on_session_reset.side_effect = lambda *a, **kw: events.append("on_session_reset")
|
||||
engine.on_session_start.side_effect = lambda *a, **kw: events.append("on_session_start")
|
||||
engine.carry_over_new_session_context.side_effect = lambda *a, **kw: events.append("carry_over")
|
||||
|
||||
agent = _bare_agent()
|
||||
agent.context_compressor = engine
|
||||
|
||||
agent._transition_context_engine_session(
|
||||
old_session_id="old-sid",
|
||||
new_session_id="new-sid",
|
||||
previous_messages=[{"role": "user", "content": "hi"}],
|
||||
carry_over_context=True,
|
||||
)
|
||||
|
||||
assert events == [
|
||||
"on_session_end",
|
||||
"on_session_reset",
|
||||
"on_session_start",
|
||||
"carry_over",
|
||||
]
|
||||
|
||||
|
||||
def test_transition_passes_conversation_id_from_gateway_session_key():
|
||||
"""on_session_start receives ``conversation_id`` from ``_gateway_session_key``."""
|
||||
engine = MagicMock()
|
||||
engine.context_length = 200_000
|
||||
captured: dict = {}
|
||||
engine.on_session_start.side_effect = lambda sid, **kw: captured.update(kw)
|
||||
|
||||
agent = _bare_agent()
|
||||
agent.context_compressor = engine
|
||||
|
||||
agent._transition_context_engine_session(
|
||||
old_session_id="old-sid",
|
||||
new_session_id="new-sid",
|
||||
previous_messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert captured.get("conversation_id") == "agent:main:telegram:dm:42"
|
||||
assert captured.get("old_session_id") == "old-sid"
|
||||
assert captured.get("platform") == "telegram"
|
||||
|
||||
|
||||
def test_transition_skips_optional_hooks_when_engine_lacks_them():
|
||||
"""Engines that don't implement on_session_end/carry_over still work."""
|
||||
class MinimalEngine:
|
||||
def __init__(self):
|
||||
self.context_length = 100_000
|
||||
self.reset_called = False
|
||||
self.start_called_with = None
|
||||
|
||||
def on_session_reset(self):
|
||||
self.reset_called = True
|
||||
|
||||
def on_session_start(self, sid, **kw):
|
||||
self.start_called_with = (sid, kw)
|
||||
|
||||
engine = MinimalEngine()
|
||||
agent = _bare_agent()
|
||||
agent.context_compressor = engine
|
||||
|
||||
# Should not raise even though on_session_end / carry_over are missing.
|
||||
agent._transition_context_engine_session(
|
||||
old_session_id="old",
|
||||
new_session_id="new",
|
||||
previous_messages=[{"role": "user", "content": "hi"}],
|
||||
carry_over_context=True,
|
||||
)
|
||||
|
||||
assert engine.reset_called is True
|
||||
assert engine.start_called_with is not None
|
||||
new_sid, kw = engine.start_called_with
|
||||
assert new_sid == "new"
|
||||
assert kw.get("old_session_id") == "old"
|
||||
|
||||
|
||||
def test_reset_session_state_delegates_to_transition_when_args_provided():
|
||||
"""``reset_session_state(previous_messages=..., old_session_id=...)`` fires full lifecycle."""
|
||||
engine = MagicMock()
|
||||
engine.context_length = 100_000
|
||||
|
||||
agent = _bare_agent()
|
||||
agent.context_compressor = engine
|
||||
|
||||
agent.reset_session_state(
|
||||
previous_messages=[{"role": "user", "content": "hi"}],
|
||||
old_session_id="old-sid",
|
||||
)
|
||||
|
||||
assert engine.on_session_end.called
|
||||
assert engine.on_session_reset.called
|
||||
assert engine.on_session_start.called
|
||||
# No carry_over_context, so carry_over hook NOT called.
|
||||
assert not engine.carry_over_new_session_context.called
|
||||
|
||||
|
||||
def test_reset_session_state_default_call_only_resets():
|
||||
"""Bare ``reset_session_state()`` still only resets the engine (no end/start)."""
|
||||
engine = MagicMock()
|
||||
engine.context_length = 100_000
|
||||
|
||||
agent = _bare_agent()
|
||||
agent.context_compressor = engine
|
||||
|
||||
agent.reset_session_state()
|
||||
|
||||
assert engine.on_session_reset.called
|
||||
assert not engine.on_session_end.called
|
||||
assert not engine.on_session_start.called
|
||||
|
||||
|
||||
def test_update_from_response_forwards_canonical_cache_buckets():
|
||||
"""conversation_loop passes cache_read/write/reasoning tokens to engine."""
|
||||
# Test the contract directly: a usage_dict built from CanonicalUsage must
|
||||
# contain the canonical buckets in addition to the legacy keys. We don't
|
||||
# spin up the full conversation loop; we just verify the dict shape.
|
||||
from agent.usage_pricing import CanonicalUsage
|
||||
|
||||
canonical = CanonicalUsage(
|
||||
input_tokens=1000,
|
||||
output_tokens=500,
|
||||
cache_read_tokens=800,
|
||||
cache_write_tokens=200,
|
||||
reasoning_tokens=50,
|
||||
)
|
||||
usage_dict = {
|
||||
"prompt_tokens": canonical.prompt_tokens,
|
||||
"completion_tokens": canonical.output_tokens,
|
||||
"total_tokens": canonical.total_tokens,
|
||||
"input_tokens": canonical.input_tokens,
|
||||
"output_tokens": canonical.output_tokens,
|
||||
"cache_read_tokens": canonical.cache_read_tokens,
|
||||
"cache_write_tokens": canonical.cache_write_tokens,
|
||||
"reasoning_tokens": canonical.reasoning_tokens,
|
||||
}
|
||||
|
||||
# Legacy keys present
|
||||
assert usage_dict["prompt_tokens"] == canonical.prompt_tokens
|
||||
assert usage_dict["completion_tokens"] == 500
|
||||
assert usage_dict["total_tokens"] == canonical.total_tokens
|
||||
# Canonical cache + reasoning buckets present
|
||||
assert usage_dict["cache_read_tokens"] == 800
|
||||
assert usage_dict["cache_write_tokens"] == 200
|
||||
assert usage_dict["reasoning_tokens"] == 50
|
||||
assert usage_dict["input_tokens"] == 1000
|
||||
assert usage_dict["output_tokens"] == 500
|
||||
|
||||
|
||||
def test_discover_context_engines_includes_plugin_registered_engines(monkeypatch):
|
||||
"""Plugin-registered context engines appear in the ``hermes plugins`` picker."""
|
||||
from hermes_cli import plugins_cmd
|
||||
|
||||
fake_repo = lambda: [("compressor", "built-in", True)]
|
||||
|
||||
class FakePluginEngine:
|
||||
name = "lcm"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.context_engine.discover_context_engines",
|
||||
fake_repo,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.discover_plugins",
|
||||
lambda *_a, **_kw: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.get_plugin_context_engine",
|
||||
lambda: FakePluginEngine(),
|
||||
)
|
||||
|
||||
engines = plugins_cmd._discover_context_engines()
|
||||
names = [n for n, _desc in engines]
|
||||
assert "compressor" in names
|
||||
assert "lcm" in names
|
||||
|
||||
|
||||
def test_discover_context_engines_dedupes_by_name(monkeypatch):
|
||||
"""Repo-shipped engine wins when name collides with a plugin-registered one."""
|
||||
from hermes_cli import plugins_cmd
|
||||
|
||||
class FakePluginEngine:
|
||||
name = "compressor" # same name as repo-shipped
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.context_engine.discover_context_engines",
|
||||
lambda: [("compressor", "built-in compressor", True)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.discover_plugins",
|
||||
lambda *_a, **_kw: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.get_plugin_context_engine",
|
||||
lambda: FakePluginEngine(),
|
||||
)
|
||||
|
||||
engines = plugins_cmd._discover_context_engines()
|
||||
# Only one entry — the repo-shipped one. Description is preserved.
|
||||
assert engines == [("compressor", "built-in compressor")]
|
||||
|
||||
|
||||
def test_engine_collector_forwards_register_command_to_plugin_manager():
|
||||
"""A plugin context engine can register a slash command via ``ctx.register_command``."""
|
||||
from plugins.context_engine import _EngineCollector
|
||||
from hermes_cli.plugins import get_plugin_manager
|
||||
|
||||
handler = lambda raw_args: f"echo: {raw_args}"
|
||||
|
||||
collector = _EngineCollector(engine_name="my-lcm")
|
||||
collector.register_command(
|
||||
"my-lcm-test-cmd",
|
||||
handler,
|
||||
description="test command from a context engine",
|
||||
args_hint="<msg>",
|
||||
)
|
||||
|
||||
manager = get_plugin_manager()
|
||||
try:
|
||||
assert "my-lcm-test-cmd" in manager._plugin_commands
|
||||
entry = manager._plugin_commands["my-lcm-test-cmd"]
|
||||
assert entry["handler"] is handler
|
||||
assert entry["args_hint"] == "<msg>"
|
||||
assert entry["plugin"] == "context-engine:my-lcm"
|
||||
finally:
|
||||
# Clean up so we don't leak the registration across tests.
|
||||
manager._plugin_commands.pop("my-lcm-test-cmd", None)
|
||||
|
||||
|
||||
def test_engine_collector_rejects_builtin_command_conflicts():
|
||||
"""Context engine cannot shadow built-in slash commands like /help."""
|
||||
from plugins.context_engine import _EngineCollector
|
||||
from hermes_cli.plugins import get_plugin_manager
|
||||
|
||||
collector = _EngineCollector(engine_name="my-lcm")
|
||||
collector.register_command("help", lambda *_: "shadow")
|
||||
|
||||
manager = get_plugin_manager()
|
||||
# Must NOT have overwritten / registered against built-in /help.
|
||||
assert "help" not in manager._plugin_commands or \
|
||||
manager._plugin_commands["help"].get("plugin") != "context-engine:my-lcm"
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_repo(tmp_path: Path) -> Path:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
_git(repo, "init")
|
||||
_git(repo, "config", "user.name", "Hermes Tests")
|
||||
_git(repo, "config", "user.email", "tests@example.com")
|
||||
|
||||
(repo / "src").mkdir()
|
||||
(repo / "src" / "main.py").write_text(
|
||||
"def alpha():\n"
|
||||
" return 'a'\n\n"
|
||||
"def beta():\n"
|
||||
" return 'b'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(repo / "src" / "helper.py").write_text("VALUE = 1\n", encoding="utf-8")
|
||||
(repo / "README.md").write_text("# Demo\n", encoding="utf-8")
|
||||
(repo / "blob.bin").write_bytes(b"\x00\x01\x02binary")
|
||||
|
||||
_git(repo, "add", ".")
|
||||
_git(repo, "commit", "-m", "initial")
|
||||
|
||||
(repo / "src" / "main.py").write_text(
|
||||
"def alpha():\n"
|
||||
" return 'changed'\n\n"
|
||||
"def beta():\n"
|
||||
" return 'b'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(repo / "src" / "helper.py").write_text("VALUE = 2\n", encoding="utf-8")
|
||||
_git(repo, "add", "src/helper.py")
|
||||
return repo
|
||||
|
||||
|
||||
def test_parse_typed_references_ignores_emails_and_handles():
|
||||
from agent.context_references import parse_context_references
|
||||
|
||||
message = (
|
||||
"email me at user@example.com and ping @teammate "
|
||||
"but include @file:src/main.py:1-2 plus @diff and @git:2 "
|
||||
"and @url:https://example.com/docs"
|
||||
)
|
||||
|
||||
refs = parse_context_references(message)
|
||||
|
||||
assert [ref.kind for ref in refs] == ["file", "diff", "git", "url"]
|
||||
assert refs[0].target == "src/main.py"
|
||||
assert refs[0].line_start == 1
|
||||
assert refs[0].line_end == 2
|
||||
assert refs[2].target == "2"
|
||||
|
||||
|
||||
def test_parse_references_strips_trailing_punctuation():
|
||||
from agent.context_references import parse_context_references
|
||||
|
||||
refs = parse_context_references(
|
||||
"review @file:README.md, then see (@url:https://example.com/docs)."
|
||||
)
|
||||
|
||||
assert [ref.kind for ref in refs] == ["file", "url"]
|
||||
assert refs[0].target == "README.md"
|
||||
assert refs[1].target == "https://example.com/docs"
|
||||
|
||||
|
||||
def test_parse_quoted_references_with_spaces_and_preserve_unquoted_ranges():
|
||||
from agent.context_references import parse_context_references
|
||||
|
||||
refs = parse_context_references(
|
||||
'review @file:"C:\\Users\\Simba\\My Project\\main.py":7-9 '
|
||||
'and @folder:"docs and specs" plus @file:src/main.py:1-2'
|
||||
)
|
||||
|
||||
assert [ref.kind for ref in refs] == ["file", "folder", "file"]
|
||||
assert refs[0].target == r"C:\Users\Simba\My Project\main.py"
|
||||
assert refs[0].line_start == 7
|
||||
assert refs[0].line_end == 9
|
||||
assert refs[1].target == "docs and specs"
|
||||
assert refs[2].target == "src/main.py"
|
||||
assert refs[2].line_start == 1
|
||||
assert refs[2].line_end == 2
|
||||
|
||||
|
||||
def test_expand_file_range_and_folder_listing(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
result = preprocess_context_references(
|
||||
"Review @file:src/main.py:1-2 and @folder:src/",
|
||||
cwd=sample_repo,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "Review and" in result.message
|
||||
assert "Review @file:src/main.py:1-2" not in result.message
|
||||
assert "--- Attached Context ---" in result.message
|
||||
assert "def alpha():" in result.message
|
||||
assert "return 'changed'" in result.message
|
||||
assert "def beta():" not in result.message
|
||||
assert "src/" in result.message
|
||||
assert "main.py" in result.message
|
||||
assert "helper.py" in result.message
|
||||
assert result.injected_tokens > 0
|
||||
assert not result.warnings
|
||||
|
||||
|
||||
def test_folder_listing_falls_back_when_rg_is_blocked(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
real_run = subprocess.run
|
||||
|
||||
def blocked_rg(*args, **kwargs):
|
||||
cmd = args[0] if args else kwargs.get("args")
|
||||
if isinstance(cmd, list) and cmd and cmd[0] == "rg":
|
||||
raise PermissionError("rg blocked by policy")
|
||||
return real_run(*args, **kwargs)
|
||||
|
||||
with patch("agent.context_references.subprocess.run", side_effect=blocked_rg):
|
||||
result = preprocess_context_references(
|
||||
"Review @folder:src/",
|
||||
cwd=sample_repo,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "src/" in result.message
|
||||
assert "main.py" in result.message
|
||||
assert "helper.py" in result.message
|
||||
assert not result.warnings
|
||||
|
||||
|
||||
def test_expand_quoted_file_reference_with_spaces(tmp_path: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
workspace = tmp_path / "repo"
|
||||
folder = workspace / "docs and specs"
|
||||
folder.mkdir(parents=True)
|
||||
file_path = folder / "release notes.txt"
|
||||
file_path.write_text("line 1\nline 2\nline 3\n", encoding="utf-8")
|
||||
|
||||
result = preprocess_context_references(
|
||||
'Review @file:"docs and specs/release notes.txt":2-3',
|
||||
cwd=workspace,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert result.message.startswith("Review")
|
||||
assert "line 1" not in result.message
|
||||
assert "line 2" in result.message
|
||||
assert "line 3" in result.message
|
||||
assert "release notes.txt" in result.message
|
||||
assert not result.warnings
|
||||
|
||||
|
||||
def test_expand_git_diff_staged_and_log(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
result = preprocess_context_references(
|
||||
"Inspect @diff and @staged and @git:1",
|
||||
cwd=sample_repo,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "git diff" in result.message
|
||||
assert "git diff --staged" in result.message
|
||||
assert "git log -1 -p" in result.message
|
||||
assert "initial" in result.message
|
||||
assert "return 'changed'" in result.message
|
||||
assert "VALUE = 2" in result.message
|
||||
|
||||
|
||||
def test_binary_and_missing_files_become_warnings(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
result = preprocess_context_references(
|
||||
"Check @file:blob.bin and @file:nope.txt",
|
||||
cwd=sample_repo,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert len(result.warnings) == 2
|
||||
assert "binary" in result.message.lower()
|
||||
assert "not found" in result.message.lower()
|
||||
|
||||
|
||||
def test_soft_budget_warns_and_hard_budget_refuses(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
soft = preprocess_context_references(
|
||||
"Check @file:src/main.py",
|
||||
cwd=sample_repo,
|
||||
context_length=100,
|
||||
)
|
||||
assert soft.expanded
|
||||
assert any("25%" in warning for warning in soft.warnings)
|
||||
|
||||
hard = preprocess_context_references(
|
||||
"Check @file:src/main.py and @file:README.md",
|
||||
cwd=sample_repo,
|
||||
context_length=20,
|
||||
)
|
||||
assert not hard.expanded
|
||||
assert hard.blocked
|
||||
assert "@file:src/main.py" in hard.message
|
||||
assert any("50%" in warning for warning in hard.warnings)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_url_expansion_uses_fetcher(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references_async
|
||||
|
||||
async def fake_fetch(url: str) -> str:
|
||||
assert url == "https://example.com/spec"
|
||||
return "# Spec\n\nImportant details."
|
||||
|
||||
result = await preprocess_context_references_async(
|
||||
"Use @url:https://example.com/spec",
|
||||
cwd=sample_repo,
|
||||
context_length=100_000,
|
||||
url_fetcher=fake_fetch,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "Important details." in result.message
|
||||
assert result.injected_tokens > 0
|
||||
|
||||
|
||||
def test_sync_url_expansion_uses_async_fetcher(sample_repo: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
async def fake_fetch(url: str) -> str:
|
||||
await asyncio.sleep(0)
|
||||
return f"Content for {url}"
|
||||
|
||||
result = preprocess_context_references(
|
||||
"Use @url:https://example.com/spec",
|
||||
cwd=sample_repo,
|
||||
context_length=100_000,
|
||||
url_fetcher=fake_fetch,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "Content for https://example.com/spec" in result.message
|
||||
|
||||
|
||||
def test_restricts_paths_to_allowed_root(tmp_path: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
(workspace / "notes.txt").write_text("inside\n", encoding="utf-8")
|
||||
secret = tmp_path / "secret.txt"
|
||||
secret.write_text("outside\n", encoding="utf-8")
|
||||
|
||||
result = preprocess_context_references(
|
||||
"read @file:../secret.txt and @file:notes.txt",
|
||||
cwd=workspace,
|
||||
context_length=100_000,
|
||||
allowed_root=workspace,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "```\noutside\n```" not in result.message
|
||||
assert "inside" in result.message
|
||||
assert any("outside the allowed workspace" in warning for warning in result.warnings)
|
||||
|
||||
|
||||
def test_defaults_allowed_root_to_cwd(tmp_path: Path):
|
||||
from agent.context_references import preprocess_context_references
|
||||
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
secret = tmp_path / "secret.txt"
|
||||
secret.write_text("outside\n", encoding="utf-8")
|
||||
|
||||
result = preprocess_context_references(
|
||||
f"read @file:{secret}",
|
||||
cwd=workspace,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "```\noutside\n```" not in result.message
|
||||
assert any("outside the allowed workspace" in warning for warning in result.warnings)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blocks_sensitive_home_and_hermes_paths(tmp_path: Path, monkeypatch):
|
||||
from agent.context_references import preprocess_context_references_async
|
||||
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
|
||||
hermes_env = tmp_path / ".hermes" / ".env"
|
||||
hermes_env.parent.mkdir(parents=True)
|
||||
hermes_env.write_text("API_KEY=super-secret\n", encoding="utf-8")
|
||||
|
||||
ssh_key = tmp_path / ".ssh" / "id_rsa"
|
||||
ssh_key.parent.mkdir(parents=True)
|
||||
ssh_key.write_text("PRIVATE-KEY\n", encoding="utf-8")
|
||||
|
||||
result = await preprocess_context_references_async(
|
||||
"read @file:.hermes/.env and @file:.ssh/id_rsa",
|
||||
cwd=tmp_path,
|
||||
allowed_root=tmp_path,
|
||||
context_length=100_000,
|
||||
)
|
||||
|
||||
assert result.expanded
|
||||
assert "API_KEY=super-secret" not in result.message
|
||||
assert "PRIVATE-KEY" not in result.message
|
||||
assert any("sensitive credential" in warning for warning in result.warnings)
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Focused regressions for the Copilot ACP shim safety layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent.copilot_acp_client import CopilotACPClient
|
||||
|
||||
|
||||
class _FakeProcess:
|
||||
def __init__(self) -> None:
|
||||
self.stdin = io.StringIO()
|
||||
|
||||
|
||||
class CopilotACPClientSafetyTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.client = CopilotACPClient(acp_cwd="/tmp")
|
||||
|
||||
def _dispatch(self, message: dict, *, cwd: str) -> dict:
|
||||
process = _FakeProcess()
|
||||
handled = self.client._handle_server_message(
|
||||
message,
|
||||
process=process,
|
||||
cwd=cwd,
|
||||
text_parts=[],
|
||||
reasoning_parts=[],
|
||||
)
|
||||
self.assertTrue(handled)
|
||||
payload = process.stdin.getvalue().strip()
|
||||
self.assertTrue(payload)
|
||||
return json.loads(payload)
|
||||
|
||||
def test_request_permission_is_not_auto_allowed(self) -> None:
|
||||
response = self._dispatch(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "session/request_permission",
|
||||
"params": {},
|
||||
},
|
||||
cwd="/tmp",
|
||||
)
|
||||
|
||||
outcome = (((response.get("result") or {}).get("outcome") or {}).get("outcome"))
|
||||
self.assertEqual(outcome, "cancelled")
|
||||
|
||||
def test_read_text_file_blocks_internal_hermes_hub_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
home = Path(tmpdir) / "home"
|
||||
blocked = home / ".hermes" / "skills" / ".hub" / "index-cache" / "entry.json"
|
||||
blocked.parent.mkdir(parents=True, exist_ok=True)
|
||||
blocked.write_text('{"token":"sk-test-secret-1234567890"}')
|
||||
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{"HOME": str(home), "HERMES_HOME": str(home / ".hermes")},
|
||||
clear=False,
|
||||
):
|
||||
response = self._dispatch(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "fs/read_text_file",
|
||||
"params": {"path": str(blocked)},
|
||||
},
|
||||
cwd=str(home),
|
||||
)
|
||||
|
||||
self.assertIn("error", response)
|
||||
|
||||
def test_read_text_file_redacts_sensitive_content(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
secret_file = root / "config.env"
|
||||
secret_file.write_text("OPENAI_API_KEY=sk-proj-abc123def456ghi789jkl012")
|
||||
|
||||
# agent.redact snapshots HERMES_REDACT_SECRETS at import time into
|
||||
# _REDACT_ENABLED, so patching os.environ is a no-op. Flip the
|
||||
# module-level constant directly for the duration of the call.
|
||||
with patch("agent.redact._REDACT_ENABLED", True):
|
||||
response = self._dispatch(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3,
|
||||
"method": "fs/read_text_file",
|
||||
"params": {"path": str(secret_file)},
|
||||
},
|
||||
cwd=str(root),
|
||||
)
|
||||
|
||||
content = ((response.get("result") or {}).get("content") or "")
|
||||
self.assertNotIn("abc123def456", content)
|
||||
self.assertIn("OPENAI_API_KEY=", content)
|
||||
|
||||
def test_write_text_file_reuses_write_denylist(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
home = Path(tmpdir) / "home"
|
||||
target = home / ".ssh" / "id_rsa"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with patch("agent.copilot_acp_client.is_write_denied", return_value=True, create=True):
|
||||
response = self._dispatch(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 4,
|
||||
"method": "fs/write_text_file",
|
||||
"params": {
|
||||
"path": str(target),
|
||||
"content": "fake-private-key",
|
||||
},
|
||||
},
|
||||
cwd=str(home),
|
||||
)
|
||||
|
||||
self.assertIn("error", response)
|
||||
self.assertFalse(target.exists())
|
||||
|
||||
def test_write_text_file_respects_safe_root(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
safe_root = root / "workspace"
|
||||
safe_root.mkdir()
|
||||
outside = root / "outside.txt"
|
||||
|
||||
with patch.dict(os.environ, {"HERMES_WRITE_SAFE_ROOT": str(safe_root)}, clear=False):
|
||||
response = self._dispatch(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 5,
|
||||
"method": "fs/write_text_file",
|
||||
"params": {
|
||||
"path": str(outside),
|
||||
"content": "should-not-write",
|
||||
},
|
||||
},
|
||||
cwd=str(root),
|
||||
)
|
||||
|
||||
self.assertIn("error", response)
|
||||
self.assertFalse(outside.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
# ── HOME env propagation tests (from PR #11285) ─────────────────────
|
||||
|
||||
from unittest.mock import patch as _patch
|
||||
import pytest
|
||||
|
||||
|
||||
def _make_home_client(tmp_path):
|
||||
return CopilotACPClient(
|
||||
api_key="copilot-acp",
|
||||
base_url="acp://copilot",
|
||||
acp_command="copilot",
|
||||
acp_args=["--acp", "--stdio"],
|
||||
acp_cwd=str(tmp_path),
|
||||
)
|
||||
|
||||
|
||||
def _fake_popen_capture(captured):
|
||||
def _fake(cmd, **kwargs):
|
||||
captured["cmd"] = cmd
|
||||
captured["kwargs"] = kwargs
|
||||
raise FileNotFoundError("copilot not found")
|
||||
return _fake
|
||||
|
||||
|
||||
def test_run_prompt_prefers_profile_home_when_available(monkeypatch, tmp_path):
|
||||
hermes_home = tmp_path / "hermes"
|
||||
profile_home = hermes_home / "home"
|
||||
profile_home.mkdir(parents=True)
|
||||
|
||||
monkeypatch.delenv("HOME", raising=False)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
captured = {}
|
||||
client = _make_home_client(tmp_path)
|
||||
|
||||
with _patch("agent.copilot_acp_client.subprocess.Popen", side_effect=_fake_popen_capture(captured)):
|
||||
with pytest.raises(RuntimeError, match="Could not start Copilot ACP command"):
|
||||
client._run_prompt("hello", timeout_seconds=1)
|
||||
|
||||
assert captured["kwargs"]["env"]["HOME"] == str(profile_home)
|
||||
|
||||
|
||||
def test_run_prompt_passes_home_when_parent_env_is_clean(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("HOME", raising=False)
|
||||
monkeypatch.delenv("HERMES_HOME", raising=False)
|
||||
|
||||
captured = {}
|
||||
client = _make_home_client(tmp_path)
|
||||
|
||||
with _patch("agent.copilot_acp_client.subprocess.Popen", side_effect=_fake_popen_capture(captured)):
|
||||
with pytest.raises(RuntimeError, match="Could not start Copilot ACP command"):
|
||||
client._run_prompt("hello", timeout_seconds=1)
|
||||
|
||||
assert "env" in captured["kwargs"]
|
||||
assert captured["kwargs"]["env"]["HOME"]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Tests for gh-copilot CLI deprecation detection and GitHub Models Azure URL mapping."""
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.copilot_acp_client import _is_gh_copilot_deprecation_message
|
||||
|
||||
|
||||
class TestDeprecationPatternDetection:
|
||||
"""Verify that stderr from the deprecated `gh copilot` extension is caught
|
||||
without false-positiving on the new `@github/copilot` CLI."""
|
||||
|
||||
_REAL_DEPRECATION_STDERR = (
|
||||
"The gh-copilot extension has been deprecated in favor of the newer "
|
||||
"GitHub Copilot CLI.\nFor more information, visit:\n"
|
||||
"- Copilot CLI: https://github.com/github/copilot-cli\n"
|
||||
"- Deprecation announcement: https://github.blog/changelog/"
|
||||
"2025-09-25-upcoming-deprecation-of-gh-copilot-cli-extension\n"
|
||||
"No commands will be executed."
|
||||
)
|
||||
|
||||
def test_real_deprecation_message_matches(self):
|
||||
assert _is_gh_copilot_deprecation_message(self._REAL_DEPRECATION_STDERR)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stderr_text",
|
||||
[
|
||||
# The deprecation banner uses both halves of the fingerprint.
|
||||
"The gh-copilot extension has been deprecated.",
|
||||
"gh-copilot: no commands will be executed.",
|
||||
# Mixed casing — match is case-insensitive.
|
||||
"The GH-Copilot Extension HAS BEEN DEPRECATED.",
|
||||
],
|
||||
)
|
||||
def test_genuine_deprecation_variants_match(self, stderr_text: str):
|
||||
assert _is_gh_copilot_deprecation_message(stderr_text)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stderr_text",
|
||||
[
|
||||
# Generic errors — no fingerprint at all.
|
||||
"Error: connection refused",
|
||||
"",
|
||||
# The NEW @github/copilot CLI's repo is github.com/github/copilot-cli.
|
||||
# Its stderr can legitimately mention "copilot-cli" or "deprecation"
|
||||
# in unrelated contexts; neither alone should trip the detector.
|
||||
"copilot-cli: failed to authenticate with the API",
|
||||
"warning: the --foo flag is scheduled for deprecation in v3",
|
||||
"See https://github.com/github/copilot-cli/issues for support",
|
||||
# Half the fingerprint without the other half.
|
||||
"gh-copilot: command not found",
|
||||
"extension has been deprecated (some other extension)",
|
||||
],
|
||||
)
|
||||
def test_does_not_false_positive(self, stderr_text: str):
|
||||
assert not _is_gh_copilot_deprecation_message(stderr_text)
|
||||
|
||||
|
||||
class TestGitHubModelsAzureUrl:
|
||||
"""Verify that the Azure GitHub Models URL is recognised."""
|
||||
|
||||
def test_url_to_provider_contains_azure_models(self):
|
||||
from agent.model_metadata import _URL_TO_PROVIDER
|
||||
|
||||
# Maps to the canonical "copilot" provider (same convention as the
|
||||
# other GitHub-family entries) — not the "github-models" alias.
|
||||
assert _URL_TO_PROVIDER.get("models.inference.ai.azure.com") == "copilot"
|
||||
|
||||
def test_is_github_models_base_url_recognises_azure(self):
|
||||
from hermes_cli.models import _is_github_models_base_url
|
||||
|
||||
assert _is_github_models_base_url("https://models.inference.ai.azure.com")
|
||||
assert _is_github_models_base_url("https://models.inference.ai.azure.com/v1/chat")
|
||||
|
||||
def test_is_github_models_base_url_still_recognises_github_ai(self):
|
||||
from hermes_cli.models import _is_github_models_base_url
|
||||
|
||||
assert _is_github_models_base_url("https://models.github.ai/inference")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,234 @@
|
||||
"""Tests for credential pool preservation through turn config and 429 recovery.
|
||||
|
||||
Covers:
|
||||
1. CLI _resolve_turn_agent_config passes credential_pool to runtime dict
|
||||
2. Gateway _resolve_turn_agent_config passes credential_pool to runtime dict
|
||||
3. Eager fallback deferred when credential pool has credentials
|
||||
4. Eager fallback fires when no credential pool exists
|
||||
5. Full 429 rotation cycle: retry-same → rotate → exhaust → fallback
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. CLI _resolve_turn_agent_config includes credential_pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCliTurnRoutePool:
|
||||
def test_resolve_turn_includes_pool(self):
|
||||
"""CLI's _resolve_turn_agent_config must pass credential_pool in runtime."""
|
||||
fake_pool = MagicMock(name="FakePool")
|
||||
shell = SimpleNamespace(
|
||||
model="gpt-5.4",
|
||||
api_key="sk-test",
|
||||
base_url=None,
|
||||
provider="openai-codex",
|
||||
api_mode="codex_responses",
|
||||
acp_command=None,
|
||||
acp_args=[],
|
||||
_credential_pool=fake_pool,
|
||||
service_tier=None,
|
||||
)
|
||||
|
||||
from cli import HermesCLI
|
||||
bound = HermesCLI._resolve_turn_agent_config.__get__(shell)
|
||||
route = bound("test message")
|
||||
|
||||
assert route["runtime"]["credential_pool"] is fake_pool
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Gateway _resolve_turn_agent_config includes credential_pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGatewayTurnRoutePool:
|
||||
def test_resolve_turn_includes_pool(self):
|
||||
"""Gateway's _resolve_turn_agent_config must pass credential_pool."""
|
||||
from gateway.run import GatewayRunner
|
||||
|
||||
fake_pool = MagicMock(name="FakePool")
|
||||
runner = SimpleNamespace(_service_tier=None)
|
||||
runtime_kwargs = {
|
||||
"api_key": "***",
|
||||
"base_url": None,
|
||||
"provider": "openai-codex",
|
||||
"api_mode": "codex_responses",
|
||||
"command": None,
|
||||
"args": [],
|
||||
"credential_pool": fake_pool,
|
||||
}
|
||||
|
||||
bound = GatewayRunner._resolve_turn_agent_config.__get__(runner)
|
||||
route = bound("test message", "gpt-5.4", runtime_kwargs)
|
||||
|
||||
assert route["runtime"]["credential_pool"] is fake_pool
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3 & 4. Eager fallback deferred/fires based on credential pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestEagerFallbackWithPool:
|
||||
"""Test the eager fallback guard in run_agent.py's error handling loop."""
|
||||
|
||||
def _make_agent(self, has_pool=True, pool_has_creds=True, has_fallback=True):
|
||||
"""Create a minimal AIAgent mock with the fields needed."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
with patch.object(AIAgent, "__init__", lambda self, **kw: None):
|
||||
agent = AIAgent()
|
||||
|
||||
agent._credential_pool = None
|
||||
if has_pool:
|
||||
pool = MagicMock()
|
||||
pool.has_available.return_value = pool_has_creds
|
||||
agent._credential_pool = pool
|
||||
|
||||
agent._fallback_chain = [{"model": "fallback/model"}] if has_fallback else []
|
||||
agent._fallback_index = 0
|
||||
agent._try_activate_fallback = MagicMock(return_value=True)
|
||||
agent._emit_status = MagicMock()
|
||||
|
||||
return agent
|
||||
|
||||
def test_eager_fallback_deferred_when_pool_has_credentials(self):
|
||||
"""429 with active pool should NOT trigger eager fallback."""
|
||||
agent = self._make_agent(has_pool=True, pool_has_creds=True, has_fallback=True)
|
||||
|
||||
# Simulate the check from run_agent.py lines 7180-7191
|
||||
is_rate_limited = True
|
||||
if is_rate_limited and agent._fallback_index < len(agent._fallback_chain):
|
||||
pool = agent._credential_pool
|
||||
pool_may_recover = pool is not None and pool.has_available()
|
||||
if not pool_may_recover:
|
||||
agent._try_activate_fallback()
|
||||
|
||||
agent._try_activate_fallback.assert_not_called()
|
||||
|
||||
def test_eager_fallback_fires_when_no_pool(self):
|
||||
"""429 without pool should trigger eager fallback."""
|
||||
agent = self._make_agent(has_pool=False, has_fallback=True)
|
||||
|
||||
is_rate_limited = True
|
||||
if is_rate_limited and agent._fallback_index < len(agent._fallback_chain):
|
||||
pool = agent._credential_pool
|
||||
pool_may_recover = pool is not None and pool.has_available()
|
||||
if not pool_may_recover:
|
||||
agent._try_activate_fallback()
|
||||
|
||||
agent._try_activate_fallback.assert_called_once()
|
||||
|
||||
def test_eager_fallback_fires_when_pool_exhausted(self):
|
||||
"""429 with exhausted pool should trigger eager fallback."""
|
||||
agent = self._make_agent(has_pool=True, pool_has_creds=False, has_fallback=True)
|
||||
|
||||
is_rate_limited = True
|
||||
if is_rate_limited and agent._fallback_index < len(agent._fallback_chain):
|
||||
pool = agent._credential_pool
|
||||
pool_may_recover = pool is not None and pool.has_available()
|
||||
if not pool_may_recover:
|
||||
agent._try_activate_fallback()
|
||||
|
||||
agent._try_activate_fallback.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Full 429 rotation cycle via _recover_with_credential_pool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPoolRotationCycle:
|
||||
"""Verify the retry-same → rotate → exhaust flow in _recover_with_credential_pool."""
|
||||
|
||||
def _make_agent_with_pool(self, pool_entries=3):
|
||||
from run_agent import AIAgent
|
||||
|
||||
with patch.object(AIAgent, "__init__", lambda self, **kw: None):
|
||||
agent = AIAgent()
|
||||
|
||||
entries = []
|
||||
for i in range(pool_entries):
|
||||
e = MagicMock(name=f"entry_{i}")
|
||||
e.id = f"cred-{i}"
|
||||
entries.append(e)
|
||||
|
||||
pool = MagicMock()
|
||||
pool.has_credentials.return_value = True
|
||||
|
||||
# mark_exhausted_and_rotate returns next entry until exhausted
|
||||
self._rotation_index = 0
|
||||
|
||||
def rotate(status_code=None, error_context=None):
|
||||
self._rotation_index += 1
|
||||
if self._rotation_index < pool_entries:
|
||||
return entries[self._rotation_index]
|
||||
pool.has_credentials.return_value = False
|
||||
return None
|
||||
|
||||
pool.mark_exhausted_and_rotate = MagicMock(side_effect=rotate)
|
||||
agent._credential_pool = pool
|
||||
agent._swap_credential = MagicMock()
|
||||
agent.log_prefix = ""
|
||||
|
||||
return agent, pool, entries
|
||||
|
||||
def test_first_429_sets_retry_flag_no_rotation(self):
|
||||
"""First 429 should just set has_retried_429=True, no rotation."""
|
||||
agent, pool, _ = self._make_agent_with_pool(3)
|
||||
recovered, has_retried = agent._recover_with_credential_pool(
|
||||
status_code=429, has_retried_429=False
|
||||
)
|
||||
assert recovered is False
|
||||
assert has_retried is True
|
||||
pool.mark_exhausted_and_rotate.assert_not_called()
|
||||
|
||||
def test_second_429_rotates_to_next(self):
|
||||
"""Second consecutive 429 should rotate to next credential."""
|
||||
agent, pool, entries = self._make_agent_with_pool(3)
|
||||
recovered, has_retried = agent._recover_with_credential_pool(
|
||||
status_code=429, has_retried_429=True
|
||||
)
|
||||
assert recovered is True
|
||||
assert has_retried is False # reset after rotation
|
||||
pool.mark_exhausted_and_rotate.assert_called_once_with(status_code=429, error_context=None)
|
||||
agent._swap_credential.assert_called_once_with(entries[1])
|
||||
|
||||
def test_pool_exhaustion_returns_false(self):
|
||||
"""When all credentials exhausted, recovery should return False."""
|
||||
agent, pool, _ = self._make_agent_with_pool(1)
|
||||
# First 429 sets flag
|
||||
_, has_retried = agent._recover_with_credential_pool(
|
||||
status_code=429, has_retried_429=False
|
||||
)
|
||||
assert has_retried is True
|
||||
|
||||
# Second 429 tries to rotate but pool is exhausted (only 1 entry)
|
||||
recovered, _ = agent._recover_with_credential_pool(
|
||||
status_code=429, has_retried_429=True
|
||||
)
|
||||
assert recovered is False
|
||||
|
||||
def test_402_immediate_rotation(self):
|
||||
"""402 (billing) should immediately rotate, no retry-first."""
|
||||
agent, pool, entries = self._make_agent_with_pool(3)
|
||||
recovered, has_retried = agent._recover_with_credential_pool(
|
||||
status_code=402, has_retried_429=False
|
||||
)
|
||||
assert recovered is True
|
||||
assert has_retried is False
|
||||
pool.mark_exhausted_and_rotate.assert_called_once_with(status_code=402, error_context=None)
|
||||
|
||||
def test_no_pool_returns_false(self):
|
||||
"""No pool should return (False, unchanged)."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
with patch.object(AIAgent, "__init__", lambda self, **kw: None):
|
||||
agent = AIAgent()
|
||||
agent._credential_pool = None
|
||||
|
||||
recovered, has_retried = agent._recover_with_credential_pool(
|
||||
status_code=429, has_retried_429=False
|
||||
)
|
||||
assert recovered is False
|
||||
assert has_retried is False
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Tests for cross-loop client cache isolation fix (#2681).
|
||||
|
||||
Verifies that _get_cached_client() returns different AsyncOpenAI clients
|
||||
when called from different event loops, preventing the httpx deadlock
|
||||
that occurs when a cached async client bound to loop A is reused on loop B.
|
||||
|
||||
This test file is self-contained and does not import the full tool chain,
|
||||
so it can run without optional dependencies like firecrawl.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal stubs so we can import _get_cached_client without the full tree
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _stub_resolve_provider_client(provider, model, async_mode, **kw):
|
||||
"""Return a unique mock client each time, simulating AsyncOpenAI creation."""
|
||||
client = MagicMock(name=f"client-{provider}-async={async_mode}")
|
||||
client.api_key = "test"
|
||||
client.base_url = kw.get("explicit_base_url", "http://localhost:8081/v1")
|
||||
return client, model or "test-model"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_client_cache():
|
||||
"""Clear the client cache before each test."""
|
||||
# We need to patch before importing
|
||||
with patch.dict("sys.modules", {}):
|
||||
pass
|
||||
# Import and clear
|
||||
import agent.auxiliary_client as ac
|
||||
ac._client_cache.clear()
|
||||
yield
|
||||
ac._client_cache.clear()
|
||||
|
||||
|
||||
class TestCrossLoopCacheIsolation:
|
||||
"""Verify async clients are cached per-event-loop, not globally."""
|
||||
|
||||
def test_same_loop_reuses_client(self):
|
||||
"""Within a single event loop, the same client should be returned."""
|
||||
from agent.auxiliary_client import _get_cached_client
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
client1, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
client2, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
|
||||
assert client1 is client2, (
|
||||
"Same loop should return the same cached client"
|
||||
)
|
||||
loop.close()
|
||||
|
||||
def test_different_loops_get_different_clients(self):
|
||||
"""Different event loops must get separate client instances."""
|
||||
from agent.auxiliary_client import _get_cached_client
|
||||
|
||||
results = {}
|
||||
|
||||
def _get_client_on_new_loop(name):
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
client, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
results[name] = (id(client), id(loop))
|
||||
# Don't close loop — simulates real usage where loops persist
|
||||
|
||||
t1 = threading.Thread(target=_get_client_on_new_loop, args=("a",))
|
||||
t2 = threading.Thread(target=_get_client_on_new_loop, args=("b",))
|
||||
t1.start(); t1.join()
|
||||
t2.start(); t2.join()
|
||||
|
||||
client_id_a, loop_id_a = results["a"]
|
||||
client_id_b, loop_id_b = results["b"]
|
||||
|
||||
assert loop_id_a != loop_id_b, "Test setup error: same loop on both threads"
|
||||
assert client_id_a != client_id_b, (
|
||||
"Different event loops got the SAME cached client — this causes "
|
||||
"httpx cross-loop deadlocks in gateway mode (#2681)"
|
||||
)
|
||||
|
||||
def test_sync_clients_not_affected(self):
|
||||
"""Sync clients (async_mode=False) should still be cached globally,
|
||||
since httpx.Client (sync) doesn't bind to an event loop."""
|
||||
from agent.auxiliary_client import _get_cached_client
|
||||
|
||||
results = {}
|
||||
|
||||
def _get_sync_client(name):
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
client, _ = _get_cached_client("custom", "m1", async_mode=False,
|
||||
base_url="http://localhost:8081/v1")
|
||||
results[name] = id(client)
|
||||
|
||||
t1 = threading.Thread(target=_get_sync_client, args=("a",))
|
||||
t2 = threading.Thread(target=_get_sync_client, args=("b",))
|
||||
t1.start(); t1.join()
|
||||
t2.start(); t2.join()
|
||||
|
||||
assert results["a"] == results["b"], (
|
||||
"Sync clients should be shared across threads (no loop binding)"
|
||||
)
|
||||
|
||||
def test_gateway_simulation_no_deadlock(self):
|
||||
"""Simulate gateway mode: _run_async spawns a thread with asyncio.run(),
|
||||
which creates a new loop. The cached client must be created on THAT loop,
|
||||
not reused from a different one."""
|
||||
from agent.auxiliary_client import _get_cached_client
|
||||
|
||||
# Simulate: first call on "gateway loop"
|
||||
gateway_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(gateway_loop)
|
||||
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
gateway_client, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
|
||||
# Simulate: _run_async spawns a thread with asyncio.run()
|
||||
worker_client_id = [None]
|
||||
def _worker():
|
||||
async def _inner():
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
client, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
worker_client_id[0] = id(client)
|
||||
asyncio.run(_inner())
|
||||
|
||||
t = threading.Thread(target=_worker)
|
||||
t.start()
|
||||
t.join()
|
||||
|
||||
assert worker_client_id[0] != id(gateway_client), (
|
||||
"Worker thread (asyncio.run) got the gateway's cached client — "
|
||||
"this is the exact cross-loop scenario that causes httpx deadlocks. "
|
||||
"The cache key must include the event loop identity (#2681)"
|
||||
)
|
||||
gateway_loop.close()
|
||||
|
||||
def test_closed_loop_client_discarded(self):
|
||||
"""A cached client whose loop has closed should be replaced."""
|
||||
from agent.auxiliary_client import _get_cached_client
|
||||
|
||||
loop1 = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop1)
|
||||
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
client1, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
|
||||
loop1.close()
|
||||
|
||||
# New loop on same thread
|
||||
loop2 = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop2)
|
||||
|
||||
with patch("agent.auxiliary_client.resolve_provider_client",
|
||||
side_effect=_stub_resolve_provider_client):
|
||||
client2, _ = _get_cached_client("custom", "m1", async_mode=True,
|
||||
base_url="http://localhost:8081/v1")
|
||||
|
||||
assert client1 is not client2, (
|
||||
"Client from closed loop should not be reused"
|
||||
)
|
||||
loop2.close()
|
||||
@@ -0,0 +1,885 @@
|
||||
"""Tests for agent/curator.py — orchestrator, idle gating, state transitions.
|
||||
|
||||
LLM spawning is never exercised here — `_run_llm_review` is monkeypatched so
|
||||
tests run fully offline and the curator module doesn't need real credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def curator_env(tmp_path, monkeypatch):
|
||||
"""Isolated HERMES_HOME + freshly reloaded curator + skill_usage modules."""
|
||||
home = tmp_path / ".hermes"
|
||||
(home / "skills").mkdir(parents=True)
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
import tools.skill_usage as usage
|
||||
importlib.reload(usage)
|
||||
import agent.curator as curator
|
||||
importlib.reload(curator)
|
||||
|
||||
# Neutralize the real LLM pass by default — tests opt in per-case.
|
||||
monkeypatch.setattr(curator, "_run_llm_review", lambda prompt: "llm-stub")
|
||||
|
||||
# Default: no config file → curator defaults. Tests can override.
|
||||
monkeypatch.setattr(curator, "_load_config", lambda: {})
|
||||
|
||||
return {"home": home, "curator": curator, "usage": usage}
|
||||
|
||||
|
||||
def _write_skill(skills_dir: Path, name: str):
|
||||
d = skills_dir / name
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / "SKILL.md").write_text(
|
||||
f"---\nname: {name}\ndescription: x\n---\n", encoding="utf-8",
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config gates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_curator_enabled_default_true(curator_env):
|
||||
assert curator_env["curator"].is_enabled() is True
|
||||
|
||||
|
||||
def test_curator_disabled_via_config(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
monkeypatch.setattr(c, "_load_config", lambda: {"enabled": False})
|
||||
assert c.is_enabled() is False
|
||||
assert c.should_run_now() is False
|
||||
|
||||
|
||||
def test_curator_defaults(curator_env):
|
||||
c = curator_env["curator"]
|
||||
assert c.get_interval_hours() == 24 * 7 # 7 days
|
||||
assert c.get_min_idle_hours() == 2
|
||||
assert c.get_stale_after_days() == 30
|
||||
assert c.get_archive_after_days() == 90
|
||||
|
||||
|
||||
def test_curator_config_overrides(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
monkeypatch.setattr(c, "_load_config", lambda: {
|
||||
"interval_hours": 12,
|
||||
"min_idle_hours": 0.5,
|
||||
"stale_after_days": 7,
|
||||
"archive_after_days": 60,
|
||||
})
|
||||
assert c.get_interval_hours() == 12
|
||||
assert c.get_min_idle_hours() == 0.5
|
||||
assert c.get_stale_after_days() == 7
|
||||
assert c.get_archive_after_days() == 60
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# should_run_now
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_first_run_defers(curator_env):
|
||||
"""The FIRST observation of the curator (fresh install, no state file)
|
||||
must NOT trigger an immediate run. The curator is designed to run after
|
||||
a full ``interval_hours`` of skill activity, not on the first background
|
||||
tick after installation. Fixes #18373.
|
||||
"""
|
||||
c = curator_env["curator"]
|
||||
# No state file — should defer and seed last_run_at.
|
||||
assert c.should_run_now() is False
|
||||
state = c.load_state()
|
||||
assert state.get("last_run_at") is not None, (
|
||||
"first observation should seed last_run_at so the interval clock "
|
||||
"starts ticking instead of firing immediately next tick"
|
||||
)
|
||||
# A second immediate call still returns False (seeded, not yet stale).
|
||||
assert c.should_run_now() is False
|
||||
|
||||
|
||||
def test_recent_run_blocks(curator_env):
|
||||
c = curator_env["curator"]
|
||||
c.save_state({
|
||||
"last_run_at": datetime.now(timezone.utc).isoformat(),
|
||||
"paused": False,
|
||||
})
|
||||
assert c.should_run_now() is False
|
||||
|
||||
|
||||
def test_old_run_eligible(curator_env):
|
||||
"""A run older than the configured interval should re-trigger. Use a
|
||||
2x-interval cushion so the test doesn't become coupled to the exact
|
||||
default — bumping DEFAULT_INTERVAL_HOURS shouldn't break it."""
|
||||
c = curator_env["curator"]
|
||||
long_ago = datetime.now(timezone.utc) - timedelta(
|
||||
hours=c.get_interval_hours() * 2
|
||||
)
|
||||
c.save_state({"last_run_at": long_ago.isoformat(), "paused": False})
|
||||
assert c.should_run_now() is True
|
||||
|
||||
|
||||
def test_paused_blocks_even_if_stale(curator_env):
|
||||
c = curator_env["curator"]
|
||||
long_ago = datetime.now(timezone.utc) - timedelta(days=30)
|
||||
c.save_state({"last_run_at": long_ago.isoformat(), "paused": True})
|
||||
assert c.should_run_now() is False
|
||||
|
||||
|
||||
def test_set_paused_roundtrip(curator_env):
|
||||
c = curator_env["curator"]
|
||||
c.set_paused(True)
|
||||
assert c.is_paused() is True
|
||||
c.set_paused(False)
|
||||
assert c.is_paused() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Automatic state transitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_unused_skill_transitions_to_stale(curator_env):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "old-skill")
|
||||
|
||||
# Record last-use well past stale_after_days (30 default)
|
||||
long_ago = (datetime.now(timezone.utc) - timedelta(days=45)).isoformat()
|
||||
data = u.load_usage()
|
||||
data["old-skill"] = u._empty_record()
|
||||
data["old-skill"]["created_by"] = "agent"
|
||||
data["old-skill"]["last_used_at"] = long_ago
|
||||
data["old-skill"]["created_at"] = long_ago
|
||||
u.save_usage(data)
|
||||
|
||||
counts = c.apply_automatic_transitions()
|
||||
assert counts["marked_stale"] == 1
|
||||
assert u.get_record("old-skill")["state"] == "stale"
|
||||
|
||||
|
||||
def test_very_old_skill_gets_archived(curator_env):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
skill_dir = _write_skill(skills_dir, "ancient")
|
||||
|
||||
super_old = (datetime.now(timezone.utc) - timedelta(days=120)).isoformat()
|
||||
data = u.load_usage()
|
||||
data["ancient"] = u._empty_record()
|
||||
data["ancient"]["created_by"] = "agent"
|
||||
data["ancient"]["last_used_at"] = super_old
|
||||
data["ancient"]["created_at"] = super_old
|
||||
u.save_usage(data)
|
||||
|
||||
counts = c.apply_automatic_transitions()
|
||||
assert counts["archived"] == 1
|
||||
assert not skill_dir.exists()
|
||||
assert (skills_dir / ".archive" / "ancient" / "SKILL.md").exists()
|
||||
assert u.get_record("ancient")["state"] == "archived"
|
||||
|
||||
|
||||
def test_pinned_skill_is_never_touched(curator_env):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "precious")
|
||||
|
||||
super_old = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat()
|
||||
data = u.load_usage()
|
||||
data["precious"] = u._empty_record()
|
||||
data["precious"]["created_by"] = "agent"
|
||||
data["precious"]["last_used_at"] = super_old
|
||||
data["precious"]["created_at"] = super_old
|
||||
data["precious"]["pinned"] = True
|
||||
u.save_usage(data)
|
||||
|
||||
counts = c.apply_automatic_transitions()
|
||||
assert counts["archived"] == 0
|
||||
assert counts["marked_stale"] == 0
|
||||
rec = u.get_record("precious")
|
||||
assert rec["state"] == "active" # untouched
|
||||
assert rec["pinned"] is True
|
||||
|
||||
|
||||
def test_stale_skill_reactivates_on_recent_use(curator_env):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "revived")
|
||||
|
||||
recent = datetime.now(timezone.utc).isoformat()
|
||||
data = u.load_usage()
|
||||
data["revived"] = u._empty_record()
|
||||
data["revived"]["created_by"] = "agent"
|
||||
data["revived"]["state"] = "stale"
|
||||
data["revived"]["last_used_at"] = recent
|
||||
data["revived"]["created_at"] = recent
|
||||
u.save_usage(data)
|
||||
|
||||
counts = c.apply_automatic_transitions()
|
||||
assert counts["reactivated"] == 1
|
||||
assert u.get_record("revived")["state"] == "active"
|
||||
|
||||
|
||||
def test_new_skill_without_last_used_not_immediately_archived(curator_env):
|
||||
"""A freshly-created skill with no use history should not get archived
|
||||
just because last_used_at is None."""
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "fresh")
|
||||
|
||||
# Bump nothing — record doesn't exist yet. Curator should create it
|
||||
# and fall back to created_at which is ~now.
|
||||
counts = c.apply_automatic_transitions()
|
||||
assert counts["archived"] == 0
|
||||
assert counts["marked_stale"] == 0
|
||||
assert (skills_dir / "fresh").exists()
|
||||
|
||||
|
||||
def test_manual_skill_is_not_auto_archived(curator_env):
|
||||
"""Manual skills can have usage records, but without the agent-created
|
||||
marker they must stay out of curator transitions."""
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
skill_dir = _write_skill(skills_dir, "manual")
|
||||
|
||||
super_old = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat()
|
||||
data = u.load_usage()
|
||||
data["manual"] = u._empty_record()
|
||||
data["manual"]["last_used_at"] = super_old
|
||||
data["manual"]["created_at"] = super_old
|
||||
u.save_usage(data)
|
||||
|
||||
counts = c.apply_automatic_transitions()
|
||||
assert counts["checked"] == 0
|
||||
assert counts["archived"] == 0
|
||||
assert skill_dir.exists()
|
||||
|
||||
|
||||
def test_bundled_skill_not_touched_by_transitions(curator_env):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "bundled")
|
||||
(skills_dir / ".bundled_manifest").write_text(
|
||||
"bundled:abc\n", encoding="utf-8",
|
||||
)
|
||||
|
||||
super_old = (datetime.now(timezone.utc) - timedelta(days=500)).isoformat()
|
||||
data = u.load_usage()
|
||||
data["bundled"] = u._empty_record()
|
||||
data["bundled"]["last_used_at"] = super_old
|
||||
u.save_usage(data)
|
||||
|
||||
counts = c.apply_automatic_transitions()
|
||||
# bundled skills are excluded from the agent-created list entirely
|
||||
assert counts["checked"] == 0
|
||||
assert (skills_dir / "bundled").exists() # never moved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_curator_review orchestration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_run_review_records_state(curator_env):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
u.mark_agent_created("a")
|
||||
|
||||
result = c.run_curator_review(synchronous=True)
|
||||
assert "started_at" in result
|
||||
state = c.load_state()
|
||||
assert state["last_run_at"] is not None
|
||||
assert state["run_count"] >= 1
|
||||
assert state["last_run_summary"] is not None
|
||||
|
||||
|
||||
def test_dry_run_does_not_advance_state(curator_env, monkeypatch):
|
||||
"""Dry-run previews must not bump last_run_at or run_count. A preview
|
||||
shouldn't defer the next scheduled real pass or look like a real run in
|
||||
`hermes curator status`. Fixes #18373.
|
||||
"""
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
u.mark_agent_created("a")
|
||||
|
||||
# Stub the LLM so the test doesn't need a provider.
|
||||
monkeypatch.setattr(
|
||||
c, "_run_llm_review",
|
||||
lambda prompt: {
|
||||
"final": "", "summary": "dry preview", "model": "", "provider": "",
|
||||
"tool_calls": [], "error": None,
|
||||
},
|
||||
)
|
||||
|
||||
c.run_curator_review(synchronous=True, dry_run=True)
|
||||
state = c.load_state()
|
||||
assert state.get("last_run_at") is None, "dry-run must not seed last_run_at"
|
||||
assert state.get("run_count", 0) == 0, "dry-run must not bump run_count"
|
||||
assert "dry-run" in (state.get("last_run_summary") or ""), (
|
||||
"dry-run summary should be labeled so status output is unambiguous"
|
||||
)
|
||||
|
||||
|
||||
def test_dry_run_injects_report_only_banner(curator_env, monkeypatch):
|
||||
"""The dry-run prompt must carry a banner instructing the LLM not to
|
||||
call any mutating tool. This is defense in depth — the caller also
|
||||
skips automatic transitions — but the LLM prompt is the only guard
|
||||
against the model calling skill_manage directly."""
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
u.mark_agent_created("a")
|
||||
|
||||
captured = {}
|
||||
def _stub(prompt):
|
||||
captured["prompt"] = prompt
|
||||
return {"final": "", "summary": "s", "model": "", "provider": "",
|
||||
"tool_calls": [], "error": None}
|
||||
monkeypatch.setattr(c, "_run_llm_review", _stub)
|
||||
|
||||
c.run_curator_review(synchronous=True, dry_run=True)
|
||||
assert "DRY-RUN" in captured["prompt"]
|
||||
assert "DO NOT" in captured["prompt"]
|
||||
|
||||
|
||||
def test_dry_run_skips_automatic_transitions(curator_env, monkeypatch):
|
||||
"""Dry-run must not call apply_automatic_transitions — the auto pass
|
||||
archives skills deterministically, and a preview must not touch the
|
||||
filesystem."""
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
u.mark_agent_created("a")
|
||||
|
||||
called = {"n": 0}
|
||||
def _explode(*_a, **_kw):
|
||||
called["n"] += 1
|
||||
return {"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0}
|
||||
monkeypatch.setattr(c, "apply_automatic_transitions", _explode)
|
||||
monkeypatch.setattr(
|
||||
c, "_run_llm_review",
|
||||
lambda p: {"final": "", "summary": "s", "model": "", "provider": "",
|
||||
"tool_calls": [], "error": None},
|
||||
)
|
||||
|
||||
c.run_curator_review(synchronous=True, dry_run=True)
|
||||
assert called["n"] == 0, "dry-run must skip apply_automatic_transitions"
|
||||
|
||||
|
||||
def test_run_review_synchronous_invokes_llm_stub(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
u.mark_agent_created("a")
|
||||
|
||||
calls = []
|
||||
def _stub(prompt):
|
||||
calls.append(prompt)
|
||||
return {
|
||||
"final": "stubbed-summary",
|
||||
"summary": "stubbed-summary",
|
||||
"model": "stub-model",
|
||||
"provider": "stub-provider",
|
||||
"tool_calls": [],
|
||||
"error": None,
|
||||
}
|
||||
monkeypatch.setattr(c, "_run_llm_review", _stub)
|
||||
|
||||
captured = []
|
||||
c.run_curator_review(on_summary=lambda s: captured.append(s), synchronous=True)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert "skill CURATOR" in calls[0] or "CURATOR" in calls[0]
|
||||
assert captured # on_summary was called
|
||||
assert any("stubbed-summary" in s for s in captured)
|
||||
|
||||
|
||||
def test_run_review_skips_llm_when_no_candidates(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
# No skills in the dir → no candidates
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
c, "_run_llm_review",
|
||||
lambda prompt: (calls.append(prompt), "never-called")[1],
|
||||
)
|
||||
|
||||
captured = []
|
||||
c.run_curator_review(on_summary=lambda s: captured.append(s), synchronous=True)
|
||||
|
||||
assert calls == [] # LLM not invoked
|
||||
assert any("skipped" in s for s in captured)
|
||||
|
||||
|
||||
def test_maybe_run_curator_respects_disabled(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
monkeypatch.setattr(c, "_load_config", lambda: {"enabled": False})
|
||||
result = c.maybe_run_curator()
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_maybe_run_curator_enforces_idle_gate(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
monkeypatch.setattr(c, "_load_config", lambda: {"min_idle_hours": 2})
|
||||
# idle less than the threshold
|
||||
result = c.maybe_run_curator(idle_for_seconds=60.0)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_maybe_run_curator_runs_when_eligible(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
u = curator_env["usage"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
u.mark_agent_created("a")
|
||||
# Seed last_run_at far in the past so the interval gate opens — the
|
||||
# "no state" path intentionally defers the first run now (#18373).
|
||||
long_ago = datetime.now(timezone.utc) - timedelta(hours=c.get_interval_hours() * 2)
|
||||
c.save_state({"last_run_at": long_ago.isoformat(), "paused": False})
|
||||
# Force idle over threshold
|
||||
result = c.maybe_run_curator(idle_for_seconds=99999.0)
|
||||
assert result is not None
|
||||
assert "started_at" in result
|
||||
|
||||
|
||||
def test_maybe_run_curator_defers_on_fresh_install(curator_env):
|
||||
"""Fresh install (no curator state file) must NOT fire the curator on
|
||||
the first gateway tick. The first observation seeds last_run_at and
|
||||
returns None. Fixes #18373."""
|
||||
c = curator_env["curator"]
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "a")
|
||||
# Infinite idle — the only thing that should block the run is the new
|
||||
# deferred-first-run gate.
|
||||
result = c.maybe_run_curator(idle_for_seconds=99999.0)
|
||||
assert result is None
|
||||
# And the next tick still defers (we seeded last_run_at to "now").
|
||||
result2 = c.maybe_run_curator(idle_for_seconds=99999.0)
|
||||
assert result2 is None
|
||||
|
||||
|
||||
def test_maybe_run_curator_swallows_exceptions(curator_env, monkeypatch):
|
||||
c = curator_env["curator"]
|
||||
|
||||
def explode():
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(c, "should_run_now", explode)
|
||||
# Must not raise
|
||||
assert c.maybe_run_curator() is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_state_file_survives_corrupt_read(curator_env):
|
||||
c = curator_env["curator"]
|
||||
c._state_file().write_text("not json", encoding="utf-8")
|
||||
# Must fall back to default, not raise
|
||||
assert c.load_state() == c._default_state()
|
||||
|
||||
|
||||
def test_state_atomic_write_no_tmp_leftovers(curator_env):
|
||||
c = curator_env["curator"]
|
||||
c.save_state({"paused": True})
|
||||
parent = c._state_file().parent
|
||||
for p in parent.iterdir():
|
||||
assert not p.name.startswith(".curator_state_"), f"tmp leftover: {p.name}"
|
||||
|
||||
|
||||
def test_state_preserves_last_report_path(curator_env):
|
||||
c = curator_env["curator"]
|
||||
c.save_state({
|
||||
"last_run_at": "2026-04-30T12:00:00+00:00",
|
||||
"last_run_summary": "ok",
|
||||
"last_report_path": "/tmp/curator-report",
|
||||
"paused": False,
|
||||
"run_count": 1,
|
||||
})
|
||||
state = c.load_state()
|
||||
assert state["last_report_path"] == "/tmp/curator-report"
|
||||
|
||||
|
||||
def test_curator_review_prompt_has_invariants():
|
||||
"""Core invariants must be in the review prompt text."""
|
||||
from agent.curator import CURATOR_REVIEW_PROMPT
|
||||
assert "MUST NOT" in CURATOR_REVIEW_PROMPT or "DO NOT" in CURATOR_REVIEW_PROMPT
|
||||
assert "bundled" in CURATOR_REVIEW_PROMPT.lower()
|
||||
assert "delete" in CURATOR_REVIEW_PROMPT.lower()
|
||||
assert "pinned" in CURATOR_REVIEW_PROMPT.lower()
|
||||
# Must describe the actions the reviewer can take. The exact vocabulary
|
||||
# has tightened over time (the umbrella-first prompt drops 'keep' as a
|
||||
# first-class decision verb, since passive keep-everything is the
|
||||
# failure mode the prompt is trying to avoid), but the core merge /
|
||||
# archive / patch trio must remain callable.
|
||||
for verb in ("patch", "archive"):
|
||||
assert verb in CURATOR_REVIEW_PROMPT.lower()
|
||||
# Must mention consolidation (possibly via "merge" or "consolidat")
|
||||
assert "consolidat" in CURATOR_REVIEW_PROMPT.lower() or "merge" in CURATOR_REVIEW_PROMPT.lower()
|
||||
|
||||
|
||||
def test_curator_review_prompt_points_at_existing_tools_only():
|
||||
"""The review prompt must rely on existing tools (skill_manage + terminal)
|
||||
and must NOT reference bespoke curator tools that are not registered
|
||||
model tools."""
|
||||
from agent.curator import CURATOR_REVIEW_PROMPT
|
||||
assert "skill_manage" in CURATOR_REVIEW_PROMPT
|
||||
assert "skills_list" in CURATOR_REVIEW_PROMPT
|
||||
assert "skill_view" in CURATOR_REVIEW_PROMPT
|
||||
assert "terminal" in CURATOR_REVIEW_PROMPT.lower()
|
||||
# These would be nice but aren't actually registered as tools — the
|
||||
# curator uses skill_manage + terminal mv instead.
|
||||
assert "archive_skill" not in CURATOR_REVIEW_PROMPT
|
||||
assert "pin_skill" not in CURATOR_REVIEW_PROMPT
|
||||
|
||||
|
||||
def test_curator_does_not_instruct_model_to_pin():
|
||||
"""Pinning is a user opt-out, not a model decision. The prompt should
|
||||
not tell the reviewer to pin skills autonomously."""
|
||||
from agent.curator import CURATOR_REVIEW_PROMPT
|
||||
# "pinned" appears in the invariant ("skip pinned skills"), but "pin"
|
||||
# as a decision verb should not.
|
||||
lines = CURATOR_REVIEW_PROMPT.split("\n")
|
||||
decision_block = "\n".join(
|
||||
l for l in lines
|
||||
if l.strip().startswith(("keep", "patch", "archive", "consolidate", "pin "))
|
||||
)
|
||||
# No standalone "pin" action line
|
||||
assert not any(l.strip().startswith("pin ") for l in lines), (
|
||||
f"Found a pin action line in:\n{decision_block}"
|
||||
)
|
||||
|
||||
|
||||
def test_curator_review_prompt_is_umbrella_first():
|
||||
"""The curator prompt must push umbrella-building / class-level thinking,
|
||||
not pair-level 'are these two the same?' analysis."""
|
||||
from agent.curator import CURATOR_REVIEW_PROMPT
|
||||
lower = CURATOR_REVIEW_PROMPT.lower()
|
||||
# Must frame the task as active umbrella-building, not a passive audit.
|
||||
assert "umbrella" in lower, (
|
||||
"must use UMBRELLA framing — the class-first abstraction the curator "
|
||||
"is designed to produce"
|
||||
)
|
||||
# Must tell the reviewer not to stop at pair-level distinctness.
|
||||
assert "class" in lower, "must reference class-level thinking"
|
||||
# Must cover the three consolidation methods explicitly
|
||||
assert "references/" in CURATOR_REVIEW_PROMPT, (
|
||||
"must name references/ as a demotion target for session-specific content"
|
||||
)
|
||||
# templates/ and scripts/ make the umbrella a real class-level skill
|
||||
assert "templates/" in CURATOR_REVIEW_PROMPT
|
||||
assert "scripts/" in CURATOR_REVIEW_PROMPT
|
||||
# Must say the counter argument: usage=0 is not a reason to skip
|
||||
assert "use_count" in CURATOR_REVIEW_PROMPT or "counter" in lower, (
|
||||
"must pre-empt the 'usage counters are zero, I can't judge' bailout"
|
||||
)
|
||||
|
||||
|
||||
def test_curator_review_prompt_preserves_skill_package_integrity():
|
||||
"""Consolidation must not flatten package skills and break linked files."""
|
||||
from agent.curator import CURATOR_REVIEW_PROMPT
|
||||
|
||||
lower = CURATOR_REVIEW_PROMPT.lower()
|
||||
assert "complete" in lower and "directory package" in lower
|
||||
assert "not a new skill root" in lower
|
||||
assert "do not flatten only skill.md" in lower
|
||||
assert "rewrite" in lower and "new paths" in lower
|
||||
assert "archive the entire original skill package unchanged" in lower
|
||||
for dirname in ("references/", "templates/", "scripts/", "assets/"):
|
||||
assert dirname in CURATOR_REVIEW_PROMPT
|
||||
|
||||
|
||||
|
||||
def test_curator_review_prompt_offers_support_file_actions():
|
||||
"""Support-file demotion (references/templates/scripts) must be one of
|
||||
the three consolidation methods, alongside merge-into-existing and
|
||||
create-new-umbrella."""
|
||||
from agent.curator import CURATOR_REVIEW_PROMPT
|
||||
# skill_manage action=write_file is how references/ are added to an
|
||||
# existing skill — this is the create-adjacent action the curator needs
|
||||
# to demote narrow siblings without touching their SKILL.md.
|
||||
assert "write_file" in CURATOR_REVIEW_PROMPT
|
||||
# Must offer creating a brand-new umbrella when no existing one fits
|
||||
assert "action=create" in CURATOR_REVIEW_PROMPT or "create a new umbrella" in CURATOR_REVIEW_PROMPT.lower()
|
||||
|
||||
|
||||
|
||||
def test_cli_unpin_refuses_bundled_skill(curator_env, capsys):
|
||||
"""hermes curator unpin must refuse bundled/hub skills too (matches pin)."""
|
||||
from hermes_cli import curator as cli
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "ship-skill")
|
||||
(skills_dir / ".bundled_manifest").write_text(
|
||||
"ship-skill:abc\n", encoding="utf-8",
|
||||
)
|
||||
|
||||
class _A:
|
||||
skill = "ship-skill"
|
||||
|
||||
rc = cli._cmd_unpin(_A())
|
||||
captured = capsys.readouterr()
|
||||
assert rc == 1
|
||||
assert "bundled" in captured.out.lower() or "hub" in captured.out.lower()
|
||||
|
||||
|
||||
def test_cli_pin_refuses_bundled_skill(curator_env, capsys):
|
||||
from hermes_cli import curator as cli
|
||||
skills_dir = curator_env["home"] / "skills"
|
||||
_write_skill(skills_dir, "ship-skill")
|
||||
(skills_dir / ".bundled_manifest").write_text(
|
||||
"ship-skill:abc\n", encoding="utf-8",
|
||||
)
|
||||
|
||||
class _A:
|
||||
skill = "ship-skill"
|
||||
|
||||
rc = cli._cmd_pin(_A())
|
||||
captured = capsys.readouterr()
|
||||
assert rc == 1
|
||||
assert "bundled" in captured.out.lower() or "hub" in captured.out.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# curator review-model resolution (canonical auxiliary.curator slot)
|
||||
#
|
||||
# Curator was unified with the rest of the aux task system in Apr 2026 so
|
||||
# `hermes model` → auxiliary picker, the dashboard Models tab, and the full
|
||||
# per-task config (timeout, base_url, api_key, extra_body) all work for it.
|
||||
# Voscko report: curator.auxiliary.{provider,model} was advertised but never
|
||||
# read. Fix wires curator through auxiliary.curator with a legacy fallback.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_review_model_defaults_to_main_when_slot_is_auto(curator_env):
|
||||
"""auxiliary.curator absent (or auto/empty) → use main model.provider/model."""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
}
|
||||
assert curator._resolve_review_model(cfg) == ("openrouter", "openai/gpt-5.5")
|
||||
|
||||
# Explicit auto/empty slot — still main model.
|
||||
cfg["auxiliary"] = {"curator": {"provider": "auto", "model": ""}}
|
||||
assert curator._resolve_review_model(cfg) == ("openrouter", "openai/gpt-5.5")
|
||||
|
||||
|
||||
def test_review_model_honors_auxiliary_curator_slot(curator_env):
|
||||
"""auxiliary.curator.{provider,model} fully set → that pair wins."""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"auxiliary": {
|
||||
"curator": {
|
||||
"provider": "openrouter",
|
||||
"model": "openai/gpt-5.4-mini",
|
||||
},
|
||||
},
|
||||
}
|
||||
assert curator._resolve_review_model(cfg) == (
|
||||
"openrouter", "openai/gpt-5.4-mini",
|
||||
)
|
||||
|
||||
|
||||
def test_review_runtime_passes_auxiliary_curator_credentials(curator_env):
|
||||
"""Per-slot api_key/base_url must ride into resolve_runtime_provider (not main-only creds)."""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"auxiliary": {
|
||||
"curator": {
|
||||
"provider": "custom",
|
||||
"model": "local-mini",
|
||||
"api_key": "sk-curator-only",
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
},
|
||||
},
|
||||
}
|
||||
binding = curator._resolve_review_runtime(cfg)
|
||||
assert binding.provider == "custom"
|
||||
assert binding.model == "local-mini"
|
||||
assert binding.explicit_api_key == "sk-curator-only"
|
||||
assert binding.explicit_base_url == "http://localhost:11434/v1"
|
||||
|
||||
|
||||
def test_review_runtime_strips_blank_aux_credentials(curator_env):
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"auxiliary": {
|
||||
"curator": {
|
||||
"provider": "openrouter",
|
||||
"model": "x/y",
|
||||
"api_key": " ",
|
||||
"base_url": "",
|
||||
},
|
||||
},
|
||||
}
|
||||
binding = curator._resolve_review_runtime(cfg)
|
||||
assert binding.explicit_api_key is None
|
||||
assert binding.explicit_base_url is None
|
||||
|
||||
|
||||
def test_review_runtime_ignores_auxiliary_credentials_when_using_main(curator_env):
|
||||
"""Falling through to main model must not pick up stray auxiliary.curator secrets."""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"auxiliary": {
|
||||
"curator": {
|
||||
"provider": "auto",
|
||||
"model": "",
|
||||
"api_key": "must-not-leak",
|
||||
"base_url": "http://curator-slot-ignored/",
|
||||
},
|
||||
},
|
||||
}
|
||||
binding = curator._resolve_review_runtime(cfg)
|
||||
assert (binding.provider, binding.model) == ("openrouter", "openai/gpt-5.5")
|
||||
assert binding.explicit_api_key is None
|
||||
assert binding.explicit_base_url is None
|
||||
|
||||
|
||||
def test_review_runtime_legacy_auxiliary_carry_credentials(curator_env, caplog):
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"curator": {
|
||||
"auxiliary": {
|
||||
"provider": "custom",
|
||||
"model": "m",
|
||||
"api_key": "legacy-key",
|
||||
"base_url": "http://legacy/v1",
|
||||
},
|
||||
},
|
||||
}
|
||||
import logging
|
||||
with caplog.at_level(logging.INFO, logger="agent.curator"):
|
||||
binding = curator._resolve_review_runtime(cfg)
|
||||
assert binding.explicit_api_key == "legacy-key"
|
||||
assert binding.explicit_base_url == "http://legacy/v1"
|
||||
assert any("deprecated curator.auxiliary" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_review_model_auxiliary_curator_partial_override_falls_back(curator_env):
|
||||
"""Only one of slot provider/model set → fall back to the main pair.
|
||||
|
||||
Prevents half-configured overrides from sending an empty side to
|
||||
resolve_runtime_provider.
|
||||
"""
|
||||
curator = curator_env["curator"]
|
||||
base_main = {"provider": "openrouter", "default": "openai/gpt-5.5"}
|
||||
|
||||
cfg_provider_only = {
|
||||
"model": dict(base_main),
|
||||
"auxiliary": {"curator": {"provider": "openrouter", "model": ""}},
|
||||
}
|
||||
assert curator._resolve_review_model(cfg_provider_only) == (
|
||||
"openrouter", "openai/gpt-5.5",
|
||||
)
|
||||
|
||||
cfg_model_only = {
|
||||
"model": dict(base_main),
|
||||
"auxiliary": {"curator": {"provider": "auto", "model": "gpt-5.4-mini"}},
|
||||
}
|
||||
assert curator._resolve_review_model(cfg_model_only) == (
|
||||
"openrouter", "openai/gpt-5.5",
|
||||
)
|
||||
|
||||
|
||||
def test_review_model_legacy_curator_auxiliary_still_works(curator_env, caplog):
|
||||
"""Pre-unification users set curator.auxiliary.{provider,model} — honor it.
|
||||
|
||||
Emits a deprecation log line but keeps their config working.
|
||||
"""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"curator": {
|
||||
"auxiliary": {
|
||||
"provider": "openrouter",
|
||||
"model": "openai/gpt-5.4-mini",
|
||||
},
|
||||
},
|
||||
}
|
||||
import logging
|
||||
with caplog.at_level(logging.INFO, logger="agent.curator"):
|
||||
result = curator._resolve_review_model(cfg)
|
||||
assert result == ("openrouter", "openai/gpt-5.4-mini")
|
||||
assert any(
|
||||
"deprecated curator.auxiliary" in rec.message for rec in caplog.records
|
||||
), "expected deprecation warning when legacy curator.auxiliary is used"
|
||||
|
||||
|
||||
def test_review_model_new_slot_wins_over_legacy(curator_env):
|
||||
"""When BOTH new and legacy are set, the canonical slot wins."""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {
|
||||
"model": {"provider": "openrouter", "default": "openai/gpt-5.5"},
|
||||
"auxiliary": {
|
||||
"curator": {"provider": "nous", "model": "new-winner"},
|
||||
},
|
||||
"curator": {
|
||||
"auxiliary": {"provider": "openrouter", "model": "legacy-loser"},
|
||||
},
|
||||
}
|
||||
assert curator._resolve_review_model(cfg) == ("nous", "new-winner")
|
||||
|
||||
|
||||
def test_review_model_handles_missing_sections(curator_env):
|
||||
"""Missing auxiliary/curator sections never raise — fall back cleanly."""
|
||||
curator = curator_env["curator"]
|
||||
cfg = {"model": {"provider": "anthropic", "model": "claude-sonnet-4-6"}}
|
||||
assert curator._resolve_review_model(cfg) == (
|
||||
"anthropic", "claude-sonnet-4-6",
|
||||
)
|
||||
|
||||
# Completely empty config → ("auto", "") — resolve_runtime_provider
|
||||
# handles the auto-detection chain from there.
|
||||
assert curator._resolve_review_model({}) == ("auto", "")
|
||||
|
||||
|
||||
def test_curator_slot_is_canonical_aux_task():
|
||||
"""Curator must be a first-class slot in every aux-task registry.
|
||||
|
||||
Four sources of truth, all checked by the shared registry test
|
||||
(test_aux_config.py) for the main tasks — this test pins `curator`
|
||||
specifically so the unification doesn't silently regress.
|
||||
"""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
from hermes_cli.main import _AUX_TASKS
|
||||
from hermes_cli.web_server import _AUX_TASK_SLOTS
|
||||
|
||||
# 1. DEFAULT_CONFIG.auxiliary — schema source
|
||||
assert "curator" in DEFAULT_CONFIG["auxiliary"], \
|
||||
"curator missing from DEFAULT_CONFIG['auxiliary']"
|
||||
slot = DEFAULT_CONFIG["auxiliary"]["curator"]
|
||||
assert slot["provider"] == "auto"
|
||||
assert slot["model"] == ""
|
||||
assert slot["timeout"] > 0, "curator timeout should be set (reviews run long)"
|
||||
|
||||
# 2. hermes_cli/main.py _AUX_TASKS — CLI picker
|
||||
aux_keys = {k for k, _name, _desc in _AUX_TASKS}
|
||||
assert "curator" in aux_keys, "curator missing from _AUX_TASKS (CLI picker)"
|
||||
|
||||
# 3. hermes_cli/web_server.py _AUX_TASK_SLOTS — REST API allowlist
|
||||
assert "curator" in _AUX_TASK_SLOTS, \
|
||||
"curator missing from _AUX_TASK_SLOTS (dashboard REST API)"
|
||||
|
||||
# 4. web/src/pages/ModelsPage.tsx is checked at build time; the tsx
|
||||
# array and this tuple share a ``Must match _AUX_TASK_SLOTS`` comment.
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Regression tests for curator skill activity timestamps."""
|
||||
|
||||
import importlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _write_skill(skills_dir: Path, name: str) -> None:
|
||||
skill_dir = skills_dir / name
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir / "SKILL.md").write_text(
|
||||
f"---\nname: {name}\ndescription: test skill\n---\n\n# {name}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def curator_modules(tmp_path, monkeypatch):
|
||||
home = tmp_path / ".hermes"
|
||||
(home / "skills").mkdir(parents=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
|
||||
import tools.skill_usage as skill_usage
|
||||
import agent.curator as curator
|
||||
|
||||
importlib.reload(skill_usage)
|
||||
importlib.reload(curator)
|
||||
return home, skill_usage, curator
|
||||
|
||||
|
||||
def test_recent_view_activity_prevents_false_stale_transition(curator_modules, monkeypatch):
|
||||
home, skill_usage, curator = curator_modules
|
||||
skills_dir = home / "skills"
|
||||
_write_skill(skills_dir, "recently-viewed")
|
||||
|
||||
now = datetime(2026, 4, 30, tzinfo=timezone.utc)
|
||||
created_at = (now - timedelta(days=60)).isoformat()
|
||||
last_viewed_at = (now - timedelta(days=1)).isoformat()
|
||||
skill_usage.save_usage({
|
||||
"recently-viewed": {
|
||||
"created_at": created_at,
|
||||
"last_viewed_at": last_viewed_at,
|
||||
"view_count": 1,
|
||||
"state": "active",
|
||||
}
|
||||
})
|
||||
monkeypatch.setattr(curator, "get_stale_after_days", lambda: 30)
|
||||
monkeypatch.setattr(curator, "get_archive_after_days", lambda: 90)
|
||||
|
||||
counts = curator.apply_automatic_transitions(now=now)
|
||||
|
||||
assert counts["marked_stale"] == 0
|
||||
assert skill_usage.get_record("recently-viewed")["state"] == "active"
|
||||
@@ -0,0 +1,594 @@
|
||||
"""Tests for agent/curator_backup.py — snapshot + rollback of the skills tree."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def backup_env(monkeypatch, tmp_path):
|
||||
"""Isolate HERMES_HOME + reload modules so every test starts clean."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
(home / "skills").mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
|
||||
# Reload so get_hermes_home picks up the env var fresh.
|
||||
import hermes_constants
|
||||
importlib.reload(hermes_constants)
|
||||
from agent import curator_backup
|
||||
importlib.reload(curator_backup)
|
||||
return {"home": home, "skills": home / "skills", "cb": curator_backup}
|
||||
|
||||
|
||||
def _write_skill(skills_dir: Path, name: str, body: str = "body") -> Path:
|
||||
d = skills_dir / name
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / "SKILL.md").write_text(
|
||||
f"---\nname: {name}\ndescription: t\nversion: 1.0\n---\n\n{body}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# snapshot_skills
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_snapshot_creates_tarball_and_manifest(backup_env):
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
_write_skill(backup_env["skills"], "beta")
|
||||
|
||||
snap = cb.snapshot_skills(reason="test")
|
||||
assert snap is not None, "snapshot should succeed with a populated skills dir"
|
||||
assert (snap / "skills.tar.gz").exists()
|
||||
manifest = json.loads((snap / "manifest.json").read_text())
|
||||
assert manifest["reason"] == "test"
|
||||
assert manifest["skill_files"] == 2
|
||||
assert manifest["archive_bytes"] > 0
|
||||
|
||||
|
||||
def test_snapshot_excludes_backups_dir_itself(backup_env):
|
||||
"""The backup must NOT contain .curator_backups/ — that would recurse
|
||||
with every subsequent snapshot and balloon disk usage."""
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
snap1 = cb.snapshot_skills(reason="first")
|
||||
assert snap1 is not None
|
||||
snap2 = cb.snapshot_skills(reason="second")
|
||||
assert snap2 is not None
|
||||
with tarfile.open(snap2 / "skills.tar.gz") as tf:
|
||||
names = tf.getnames()
|
||||
assert not any(n.startswith(".curator_backups") for n in names), (
|
||||
"second snapshot must not contain the first snapshot recursively"
|
||||
)
|
||||
|
||||
|
||||
def test_snapshot_excludes_hub_dir(backup_env):
|
||||
""".hub/ is managed by the skills hub. Rolling it back would break
|
||||
lockfile invariants, so the snapshot omits it entirely."""
|
||||
cb = backup_env["cb"]
|
||||
hub = backup_env["skills"] / ".hub"
|
||||
hub.mkdir()
|
||||
(hub / "lock.json").write_text("{}")
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
snap = cb.snapshot_skills(reason="t")
|
||||
assert snap is not None
|
||||
with tarfile.open(snap / "skills.tar.gz") as tf:
|
||||
names = tf.getnames()
|
||||
assert not any(n.startswith(".hub") for n in names)
|
||||
|
||||
|
||||
def test_snapshot_disabled_returns_none(backup_env, monkeypatch):
|
||||
cb = backup_env["cb"]
|
||||
monkeypatch.setattr(cb, "is_enabled", lambda: False)
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
assert cb.snapshot_skills() is None
|
||||
# And no backup dir should have been created
|
||||
assert not (backup_env["skills"] / ".curator_backups").exists()
|
||||
|
||||
|
||||
def test_snapshot_uniquifies_when_same_second(backup_env, monkeypatch):
|
||||
"""Two snapshots in the same wallclock second must not clobber each
|
||||
other. The module appends a counter to the second snapshot's id."""
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
frozen = "2026-05-01T12-00-00Z"
|
||||
monkeypatch.setattr(cb, "_utc_id", lambda now=None: frozen)
|
||||
s1 = cb.snapshot_skills(reason="a")
|
||||
s2 = cb.snapshot_skills(reason="b")
|
||||
assert s1 is not None and s2 is not None
|
||||
assert s1.name == frozen
|
||||
assert s2.name == f"{frozen}-01"
|
||||
|
||||
|
||||
def test_snapshot_prunes_to_keep_count(backup_env, monkeypatch):
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
monkeypatch.setattr(cb, "get_keep", lambda: 3)
|
||||
|
||||
# Create 5 snapshots with monotonically increasing fake ids
|
||||
ids = [f"2026-05-0{i}T00-00-00Z" for i in range(1, 6)]
|
||||
for i, fid in enumerate(ids):
|
||||
monkeypatch.setattr(cb, "_utc_id", lambda now=None, _f=fid: _f)
|
||||
cb.snapshot_skills(reason=f"n{i}")
|
||||
|
||||
remaining = sorted(p.name for p in (backup_env["skills"] / ".curator_backups").iterdir())
|
||||
# Newest 3 kept (lex order == date order for this id format)
|
||||
assert remaining == ids[2:], f"expected newest 3, got {remaining}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list_backups / _resolve_backup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_list_backups_empty(backup_env):
|
||||
cb = backup_env["cb"]
|
||||
assert cb.list_backups() == []
|
||||
|
||||
|
||||
def test_list_backups_returns_manifest_data(backup_env):
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
cb.snapshot_skills(reason="m1")
|
||||
rows = cb.list_backups()
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["reason"] == "m1"
|
||||
assert rows[0]["skill_files"] == 1
|
||||
|
||||
|
||||
def test_resolve_backup_newest_when_no_id(backup_env, monkeypatch):
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
ids = ["2026-05-01T00-00-00Z", "2026-05-02T00-00-00Z"]
|
||||
for fid in ids:
|
||||
monkeypatch.setattr(cb, "_utc_id", lambda now=None, _f=fid: _f)
|
||||
cb.snapshot_skills()
|
||||
resolved = cb._resolve_backup(None)
|
||||
assert resolved is not None
|
||||
assert resolved.name == "2026-05-02T00-00-00Z", (
|
||||
"resolve(None) must return newest regular snapshot"
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_backup_unknown_id_returns_none(backup_env):
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
cb.snapshot_skills()
|
||||
assert cb._resolve_backup("not-an-id") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# rollback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_rollback_restores_deleted_skill(backup_env):
|
||||
"""The whole point of this feature: user loses a skill, rollback
|
||||
brings it back."""
|
||||
cb = backup_env["cb"]
|
||||
skills = backup_env["skills"]
|
||||
user_skill = _write_skill(skills, "my-personal-workflow", body="important content")
|
||||
cb.snapshot_skills(reason="pre-simulated-curator")
|
||||
|
||||
# Simulate curator archiving it out of existence
|
||||
import shutil as _sh
|
||||
_sh.rmtree(user_skill)
|
||||
assert not user_skill.exists()
|
||||
|
||||
ok, msg, _ = cb.rollback()
|
||||
assert ok, f"rollback failed: {msg}"
|
||||
assert user_skill.exists(), "my-personal-workflow should be restored"
|
||||
assert "important content" in (user_skill / "SKILL.md").read_text()
|
||||
|
||||
|
||||
def test_rollback_is_itself_undoable(backup_env):
|
||||
"""A rollback creates its own safety snapshot before replacing the
|
||||
tree, so the user can undo a mistaken rollback. The safety snapshot
|
||||
is a real tarball with reason='pre-rollback to <id>' — it's
|
||||
listed by list_backups() just like any other snapshot and can be
|
||||
restored the same way."""
|
||||
cb = backup_env["cb"]
|
||||
skills = backup_env["skills"]
|
||||
_write_skill(skills, "v1")
|
||||
cb.snapshot_skills(reason="snapshot-of-v1")
|
||||
|
||||
# Overwrite with a new skill state
|
||||
import shutil as _sh
|
||||
_sh.rmtree(skills / "v1")
|
||||
_write_skill(skills, "v2")
|
||||
|
||||
ok, _, _ = cb.rollback()
|
||||
assert ok
|
||||
assert (skills / "v1").exists()
|
||||
|
||||
# list_backups should show a safety snapshot tagged "pre-rollback to <target-id>"
|
||||
rows = cb.list_backups()
|
||||
pre_rollback_entries = [r for r in rows if "pre-rollback" in (r.get("reason") or "")]
|
||||
assert len(pre_rollback_entries) >= 1, (
|
||||
f"expected a pre-rollback safety snapshot in list_backups(), got: "
|
||||
f"{[(r.get('id'), r.get('reason')) for r in rows]}"
|
||||
)
|
||||
# And the transient staging dir must be gone (it's implementation detail)
|
||||
backups_dir = skills / ".curator_backups"
|
||||
staging_dirs = [p for p in backups_dir.iterdir() if p.name.startswith(".rollback-staging-")]
|
||||
assert staging_dirs == [], (
|
||||
f"staging dir should be cleaned up on success, got: {staging_dirs}"
|
||||
)
|
||||
|
||||
|
||||
def test_rollback_no_snapshots_returns_error(backup_env):
|
||||
cb = backup_env["cb"]
|
||||
ok, msg, _ = cb.rollback()
|
||||
assert not ok
|
||||
assert "no matching backup" in msg.lower() or "no snapshot" in msg.lower()
|
||||
|
||||
|
||||
def test_rollback_rejects_unsafe_tarball(backup_env, monkeypatch):
|
||||
"""Tarballs with absolute paths or .. components must be refused even
|
||||
if someone crafts a malicious snapshot. Defense in depth — normal
|
||||
curator snapshots never produce these."""
|
||||
cb = backup_env["cb"]
|
||||
skills = backup_env["skills"]
|
||||
_write_skill(skills, "alpha")
|
||||
cb.snapshot_skills(reason="legit")
|
||||
|
||||
# Hand-craft a malicious tarball replacing the legit one
|
||||
rows = cb.list_backups()
|
||||
snap_dir = Path(rows[0]["path"])
|
||||
mal = snap_dir / "skills.tar.gz"
|
||||
mal.unlink()
|
||||
with tarfile.open(mal, "w:gz") as tf:
|
||||
evil = tempfile.NamedTemporaryFile(delete=False, suffix=".md")
|
||||
evil.write(b"evil")
|
||||
evil.close()
|
||||
tf.add(evil.name, arcname="../../etc/evil.md")
|
||||
os.unlink(evil.name)
|
||||
|
||||
ok, msg, _ = cb.rollback()
|
||||
assert not ok
|
||||
assert "unsafe" in msg.lower() or "refus" in msg.lower() or "extract" in msg.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration with run_curator_review
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_real_run_takes_pre_snapshot(backup_env, monkeypatch):
|
||||
"""A real (non-dry) curator pass must snapshot the tree before calling
|
||||
apply_automatic_transitions. This is the safety net #18373 asked for."""
|
||||
cb = backup_env["cb"]
|
||||
skills = backup_env["skills"]
|
||||
_write_skill(skills, "alpha")
|
||||
|
||||
# Reload curator module against the freshly-env'd hermes_constants
|
||||
from agent import curator
|
||||
importlib.reload(curator)
|
||||
|
||||
# Stub out LLM review and auto transitions — we only care about the
|
||||
# snapshot side-effect.
|
||||
monkeypatch.setattr(
|
||||
curator, "_run_llm_review",
|
||||
lambda p: {"final": "", "summary": "s", "model": "", "provider": "",
|
||||
"tool_calls": [], "error": None},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
curator, "apply_automatic_transitions",
|
||||
lambda now=None: {"checked": 1, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
||||
)
|
||||
|
||||
curator.run_curator_review(synchronous=True)
|
||||
# Pre-run snapshot should exist
|
||||
rows = cb.list_backups()
|
||||
assert any(r.get("reason") == "pre-curator-run" for r in rows), (
|
||||
f"expected a pre-curator-run snapshot, got {[r.get('reason') for r in rows]}"
|
||||
)
|
||||
|
||||
|
||||
def test_dry_run_skips_snapshot(backup_env, monkeypatch):
|
||||
"""Dry-run previews must not spend disk on a snapshot — they don't
|
||||
mutate anything, so there's nothing to back up."""
|
||||
cb = backup_env["cb"]
|
||||
skills = backup_env["skills"]
|
||||
_write_skill(skills, "alpha")
|
||||
|
||||
from agent import curator
|
||||
importlib.reload(curator)
|
||||
monkeypatch.setattr(
|
||||
curator, "_run_llm_review",
|
||||
lambda p: {"final": "", "summary": "s", "model": "", "provider": "",
|
||||
"tool_calls": [], "error": None},
|
||||
)
|
||||
|
||||
curator.run_curator_review(synchronous=True, dry_run=True)
|
||||
rows = cb.list_backups()
|
||||
assert not any(r.get("reason") == "pre-curator-run" for r in rows), (
|
||||
"dry-run must not create a pre-run snapshot"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cron-jobs backup + rollback (the part issue #18671's follow-up adds)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_cron_jobs(home: Path, jobs: list) -> Path:
|
||||
"""Write a synthetic cron/jobs.json under HERMES_HOME. Returns the path.
|
||||
Mirrors cron.jobs.save_jobs() wrapper shape: `{"jobs": [...], "updated_at": ...}`.
|
||||
"""
|
||||
cron_dir = home / "cron"
|
||||
cron_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = cron_dir / "jobs.json"
|
||||
path.write_text(
|
||||
json.dumps({"jobs": jobs, "updated_at": "2026-05-01T00:00:00Z"}, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _reload_cron_jobs(home: Path):
|
||||
"""Reload cron.jobs so its module-level HERMES_DIR picks up the tmp HOME."""
|
||||
import hermes_constants
|
||||
importlib.reload(hermes_constants)
|
||||
if "cron.jobs" in sys.modules:
|
||||
import cron.jobs as _cj
|
||||
importlib.reload(_cj)
|
||||
else:
|
||||
import cron.jobs as _cj # noqa: F401
|
||||
import cron.jobs as cj
|
||||
return cj
|
||||
|
||||
|
||||
def test_snapshot_includes_cron_jobs(backup_env):
|
||||
"""With a cron/jobs.json present, snapshot writes cron-jobs.json and records it in manifest."""
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
_write_cron_jobs(backup_env["home"], [
|
||||
{"id": "job-a", "name": "a", "schedule": "every 1h", "skills": ["alpha"]},
|
||||
{"id": "job-b", "name": "b", "schedule": "every 2h", "skill": "alpha"},
|
||||
])
|
||||
|
||||
snap = cb.snapshot_skills(reason="test")
|
||||
assert snap is not None
|
||||
assert (snap / cb.CRON_JOBS_FILENAME).exists()
|
||||
|
||||
mf = json.loads((snap / "manifest.json").read_text(encoding="utf-8"))
|
||||
assert mf["cron_jobs"]["backed_up"] is True
|
||||
assert mf["cron_jobs"]["jobs_count"] == 2
|
||||
|
||||
|
||||
def test_snapshot_without_cron_jobs_file_still_succeeds(backup_env):
|
||||
"""No cron/jobs.json on disk → snapshot succeeds, manifest records absence."""
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
# Deliberately do not create ~/.hermes/cron/jobs.json
|
||||
|
||||
snap = cb.snapshot_skills(reason="test")
|
||||
assert snap is not None
|
||||
assert not (snap / cb.CRON_JOBS_FILENAME).exists()
|
||||
|
||||
mf = json.loads((snap / "manifest.json").read_text(encoding="utf-8"))
|
||||
assert mf["cron_jobs"]["backed_up"] is False
|
||||
assert "cron/jobs.json" in mf["cron_jobs"]["reason"]
|
||||
|
||||
|
||||
def test_snapshot_cron_jobs_malformed_json_still_captured(backup_env):
|
||||
"""Malformed jobs.json is still copied to the snapshot (fidelity over
|
||||
validation); the manifest notes the parse warning."""
|
||||
cb = backup_env["cb"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
(backup_env["home"] / "cron").mkdir()
|
||||
(backup_env["home"] / "cron" / "jobs.json").write_text("{oh no", encoding="utf-8")
|
||||
|
||||
snap = cb.snapshot_skills(reason="test")
|
||||
assert snap is not None
|
||||
# Raw file was copied even though we couldn't parse it
|
||||
assert (snap / cb.CRON_JOBS_FILENAME).read_text() == "{oh no"
|
||||
|
||||
mf = json.loads((snap / "manifest.json").read_text(encoding="utf-8"))
|
||||
assert mf["cron_jobs"]["backed_up"] is True
|
||||
assert mf["cron_jobs"]["jobs_count"] == 0
|
||||
assert "parse_warning" in mf["cron_jobs"]
|
||||
|
||||
|
||||
def test_rollback_restores_cron_skill_links(backup_env):
|
||||
"""End-to-end: snapshot with job [alpha,beta], curator-style in-place
|
||||
rewrite to [umbrella], then rollback → skills restored to [alpha,beta]."""
|
||||
cb = backup_env["cb"]
|
||||
home = backup_env["home"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
_write_skill(backup_env["skills"], "beta")
|
||||
_write_skill(backup_env["skills"], "umbrella")
|
||||
|
||||
cj = _reload_cron_jobs(home)
|
||||
cj.create_job(name="weekly", prompt="p", schedule="every 7d",
|
||||
skills=["alpha", "beta"])
|
||||
|
||||
snap = cb.snapshot_skills(reason="pre-curator-run")
|
||||
assert snap is not None
|
||||
|
||||
# Simulate the curator's in-place cron rewrite after consolidation
|
||||
cj.rewrite_skill_refs(
|
||||
consolidated={"alpha": "umbrella", "beta": "umbrella"},
|
||||
pruned=[],
|
||||
)
|
||||
live_after_curator = cj.load_jobs()
|
||||
assert live_after_curator[0]["skills"] == ["umbrella"]
|
||||
|
||||
# Now roll back
|
||||
ok, msg, _ = cb.rollback(backup_id=snap.name)
|
||||
assert ok, msg
|
||||
assert "cron links" in msg
|
||||
|
||||
live_after_rollback = cj.load_jobs()
|
||||
# skills restored; legacy `skill` mirror follows first element
|
||||
assert live_after_rollback[0]["skills"] == ["alpha", "beta"]
|
||||
|
||||
|
||||
def test_rollback_only_touches_skill_fields(backup_env):
|
||||
"""Every field other than skills/skill must remain untouched across rollback.
|
||||
Schedule, enabled, prompt, timestamps — all live state, hands off."""
|
||||
cb = backup_env["cb"]
|
||||
home = backup_env["home"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
|
||||
# Hand-rolled jobs.json with varied fields (no real create_job — we want
|
||||
# exact field control).
|
||||
_write_cron_jobs(home, [{
|
||||
"id": "stable-id",
|
||||
"name": "original-name",
|
||||
"prompt": "original prompt",
|
||||
"schedule": "every 1h",
|
||||
"skills": ["alpha"],
|
||||
"enabled": True,
|
||||
"last_run_at": "2026-04-01T00:00:00Z",
|
||||
}])
|
||||
snap = cb.snapshot_skills(reason="pre-curator-run")
|
||||
assert snap is not None
|
||||
|
||||
# User/scheduler activity AFTER the snapshot: rename the job, change
|
||||
# the schedule, update timestamps, and (curator) rewrite the skills list.
|
||||
cj = _reload_cron_jobs(home)
|
||||
jobs = cj.load_jobs()
|
||||
jobs[0]["name"] = "renamed-since-snapshot"
|
||||
jobs[0]["schedule"] = "every 30m"
|
||||
jobs[0]["last_run_at"] = "2026-05-01T12:00:00Z"
|
||||
jobs[0]["skills"] = ["umbrella"] # pretend curator did this
|
||||
cj.save_jobs(jobs)
|
||||
|
||||
ok, _, _ = cb.rollback(backup_id=snap.name)
|
||||
assert ok
|
||||
|
||||
after = cj.load_jobs()
|
||||
job = after[0]
|
||||
# skills: restored
|
||||
assert job["skills"] == ["alpha"]
|
||||
# everything else: untouched (live state preserved)
|
||||
assert job["name"] == "renamed-since-snapshot"
|
||||
assert job["schedule"] == "every 30m"
|
||||
assert job["last_run_at"] == "2026-05-01T12:00:00Z"
|
||||
assert job["prompt"] == "original prompt"
|
||||
|
||||
|
||||
def test_rollback_skips_jobs_the_user_deleted(backup_env):
|
||||
"""If the user deleted a cron job after the snapshot, rollback must
|
||||
NOT resurrect it — the user's delete is a later, explicit choice."""
|
||||
cb = backup_env["cb"]
|
||||
home = backup_env["home"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
|
||||
_write_cron_jobs(home, [
|
||||
{"id": "keep-me", "name": "keep", "schedule": "every 1h", "skills": ["alpha"]},
|
||||
{"id": "delete-me", "name": "gone", "schedule": "every 1h", "skills": ["alpha"]},
|
||||
])
|
||||
snap = cb.snapshot_skills(reason="pre-curator-run")
|
||||
|
||||
# User deletes one job after the snapshot
|
||||
cj = _reload_cron_jobs(home)
|
||||
cj.save_jobs([j for j in cj.load_jobs() if j["id"] != "delete-me"])
|
||||
|
||||
ok, _, _ = cb.rollback(backup_id=snap.name)
|
||||
assert ok
|
||||
|
||||
live_after = cj.load_jobs()
|
||||
live_ids = {j["id"] for j in live_after}
|
||||
assert "keep-me" in live_ids
|
||||
assert "delete-me" not in live_ids # not resurrected
|
||||
|
||||
|
||||
def test_rollback_leaves_new_jobs_untouched(backup_env):
|
||||
"""Jobs created AFTER the snapshot must pass through rollback unchanged."""
|
||||
cb = backup_env["cb"]
|
||||
home = backup_env["home"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
_write_cron_jobs(home, [
|
||||
{"id": "original", "name": "o", "schedule": "every 1h", "skills": ["alpha"]},
|
||||
])
|
||||
snap = cb.snapshot_skills(reason="pre-curator-run")
|
||||
|
||||
cj = _reload_cron_jobs(home)
|
||||
jobs = cj.load_jobs()
|
||||
jobs.append({"id": "new-after-snapshot", "name": "new",
|
||||
"schedule": "every 15m", "skills": ["brand-new-skill"]})
|
||||
cj.save_jobs(jobs)
|
||||
|
||||
ok, _, _ = cb.rollback(backup_id=snap.name)
|
||||
assert ok
|
||||
|
||||
live = cj.load_jobs()
|
||||
by_id = {j["id"]: j for j in live}
|
||||
assert "new-after-snapshot" in by_id
|
||||
# New job's fields completely preserved
|
||||
assert by_id["new-after-snapshot"]["skills"] == ["brand-new-skill"]
|
||||
assert by_id["new-after-snapshot"]["schedule"] == "every 15m"
|
||||
|
||||
|
||||
def test_rollback_with_snapshot_missing_cron_succeeds(backup_env):
|
||||
"""Older snapshots (created before this feature shipped) have no
|
||||
cron-jobs.json. Rollback must still restore the skills tree and not
|
||||
error out."""
|
||||
cb = backup_env["cb"]
|
||||
home = backup_env["home"]
|
||||
_write_skill(backup_env["skills"], "alpha")
|
||||
|
||||
# No cron/jobs.json at snapshot time — simulates a pre-feature snapshot
|
||||
snap = cb.snapshot_skills(reason="test")
|
||||
assert snap is not None
|
||||
assert not (snap / cb.CRON_JOBS_FILENAME).exists()
|
||||
|
||||
# Later the user created a cron job
|
||||
_write_cron_jobs(home, [
|
||||
{"id": "later-job", "name": "l", "schedule": "every 1h", "skills": ["x"]},
|
||||
])
|
||||
|
||||
ok, msg, _ = cb.rollback(backup_id=snap.name)
|
||||
# Main rollback still succeeds; cron report notes the missing file.
|
||||
assert ok, msg
|
||||
# Jobs.json untouched (nothing to restore from)
|
||||
cj = _reload_cron_jobs(home)
|
||||
jobs = cj.load_jobs()
|
||||
assert jobs[0]["id"] == "later-job"
|
||||
assert jobs[0]["skills"] == ["x"]
|
||||
|
||||
|
||||
def test_restore_cron_skill_links_standalone(backup_env):
|
||||
"""Unit-level test on _restore_cron_skill_links without the full rollback.
|
||||
Verifies the report structure carefully."""
|
||||
cb = backup_env["cb"]
|
||||
home = backup_env["home"]
|
||||
|
||||
# Prime a snapshot dir manually with cron-jobs.json
|
||||
backups_dir = home / "skills" / ".curator_backups" / "fake-id"
|
||||
backups_dir.mkdir(parents=True)
|
||||
(backups_dir / cb.CRON_JOBS_FILENAME).write_text(json.dumps([
|
||||
{"id": "job-1", "name": "one", "skills": ["narrow-a", "narrow-b"]},
|
||||
{"id": "job-2", "name": "two", "skill": "legacy-single"},
|
||||
{"id": "job-gone", "name": "deleted", "skills": ["whatever"]},
|
||||
]), encoding="utf-8")
|
||||
|
||||
# Live jobs: job-1 got rewritten, job-2 unchanged, job-gone deleted
|
||||
_write_cron_jobs(home, [
|
||||
{"id": "job-1", "name": "one", "skills": ["umbrella"], "schedule": "every 1h"},
|
||||
{"id": "job-2", "name": "two", "skill": "legacy-single", "schedule": "every 1h"},
|
||||
{"id": "job-new", "name": "new", "skills": ["x"], "schedule": "every 1h"},
|
||||
])
|
||||
_reload_cron_jobs(home)
|
||||
|
||||
report = cb._restore_cron_skill_links(backups_dir)
|
||||
assert report["attempted"] is True
|
||||
assert report["error"] is None
|
||||
assert report["unchanged"] == 1 # job-2 matched
|
||||
assert len(report["restored"]) == 1 # job-1 got restored
|
||||
assert report["restored"][0]["job_id"] == "job-1"
|
||||
assert report["restored"][0]["to"]["skills"] == ["narrow-a", "narrow-b"]
|
||||
assert len(report["skipped_missing"]) == 1
|
||||
assert report["skipped_missing"][0]["job_id"] == "job-gone"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,435 @@
|
||||
"""Tests for the curator per-run report writer (run.json + REPORT.md).
|
||||
|
||||
Reports live under ``~/.hermes/logs/curator/{YYYYMMDD-HHMMSS}/`` alongside
|
||||
the standard log dir, not inside the user's ``skills/`` data directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def curator_env(tmp_path, monkeypatch):
|
||||
"""Isolated HERMES_HOME with a skills/ dir + reset curator module state."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
(home / "skills").mkdir()
|
||||
(home / "logs").mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
|
||||
import importlib
|
||||
import hermes_constants
|
||||
importlib.reload(hermes_constants)
|
||||
from agent import curator
|
||||
importlib.reload(curator)
|
||||
from tools import skill_usage
|
||||
importlib.reload(skill_usage)
|
||||
yield {"home": home, "curator": curator, "skill_usage": skill_usage}
|
||||
|
||||
|
||||
def _make_llm_meta(**overrides):
|
||||
base = {
|
||||
"final": "short summary of the pass",
|
||||
"summary": "short summary",
|
||||
"model": "test-model",
|
||||
"provider": "test-provider",
|
||||
"tool_calls": [],
|
||||
"error": None,
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def test_reports_root_is_under_logs_not_skills(curator_env):
|
||||
"""Reports live in logs/curator/, not skills/ — operational telemetry
|
||||
belongs with the logs, not with user-authored skill data."""
|
||||
curator = curator_env["curator"]
|
||||
root = curator._reports_root()
|
||||
home = curator_env["home"]
|
||||
# Must be under logs/
|
||||
assert root == home / "logs" / "curator"
|
||||
# Must NOT be under skills/
|
||||
assert "skills" not in root.parts
|
||||
|
||||
|
||||
def test_write_run_report_creates_both_files(curator_env):
|
||||
"""Each run writes both a run.json (machine) and a REPORT.md (human)."""
|
||||
curator = curator_env["curator"]
|
||||
start = datetime.now(timezone.utc)
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=start,
|
||||
elapsed_seconds=12.345,
|
||||
auto_counts={"checked": 5, "marked_stale": 1, "archived": 0, "reactivated": 0},
|
||||
auto_summary="1 marked stale",
|
||||
before_report=[],
|
||||
before_names=set(),
|
||||
after_report=[],
|
||||
llm_meta=_make_llm_meta(),
|
||||
)
|
||||
assert run_dir is not None
|
||||
assert run_dir.is_dir()
|
||||
assert (run_dir / "run.json").exists()
|
||||
assert (run_dir / "REPORT.md").exists()
|
||||
|
||||
# The directory name is a timestamp under logs/curator/
|
||||
assert run_dir.parent == curator._reports_root()
|
||||
|
||||
|
||||
def test_run_json_has_expected_shape(curator_env):
|
||||
"""run.json must carry the machine-readable fields downstream tooling needs."""
|
||||
curator = curator_env["curator"]
|
||||
start = datetime.now(timezone.utc)
|
||||
|
||||
before_report = [
|
||||
{"name": "old-thing", "state": "active", "pinned": False},
|
||||
{"name": "keeper", "state": "active", "pinned": True},
|
||||
]
|
||||
after_report = [
|
||||
{"name": "keeper", "state": "active", "pinned": True},
|
||||
{"name": "new-umbrella", "state": "active", "pinned": False},
|
||||
]
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=start,
|
||||
elapsed_seconds=42.0,
|
||||
auto_counts={"checked": 2, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
||||
auto_summary="no changes",
|
||||
before_report=before_report,
|
||||
before_names={r["name"] for r in before_report},
|
||||
after_report=after_report,
|
||||
llm_meta=_make_llm_meta(
|
||||
final="I consolidated the whole universe.",
|
||||
tool_calls=[
|
||||
{"name": "skills_list", "arguments": "{}"},
|
||||
{"name": "skill_manage", "arguments": '{"action":"create"}'},
|
||||
{"name": "terminal", "arguments": "mv ..."},
|
||||
],
|
||||
),
|
||||
)
|
||||
payload = json.loads((run_dir / "run.json").read_text())
|
||||
|
||||
# top-level shape
|
||||
for k in (
|
||||
"started_at", "duration_seconds", "model", "provider",
|
||||
"auto_transitions", "counts", "tool_call_counts",
|
||||
"archived", "added", "state_transitions",
|
||||
"llm_final", "llm_summary", "llm_error", "tool_calls",
|
||||
):
|
||||
assert k in payload, f"missing key: {k}"
|
||||
|
||||
# Diff logic
|
||||
assert payload["archived"] == ["old-thing"]
|
||||
assert payload["added"] == ["new-umbrella"]
|
||||
# Counts reflect the diff
|
||||
assert payload["counts"]["before"] == 2
|
||||
assert payload["counts"]["after"] == 2
|
||||
assert payload["counts"]["archived_this_run"] == 1
|
||||
assert payload["counts"]["added_this_run"] == 1
|
||||
# Tool call counts are aggregated
|
||||
assert payload["tool_call_counts"]["skills_list"] == 1
|
||||
assert payload["tool_call_counts"]["skill_manage"] == 1
|
||||
assert payload["tool_call_counts"]["terminal"] == 1
|
||||
assert payload["counts"]["tool_calls_total"] == 3
|
||||
|
||||
|
||||
def test_report_md_is_human_readable(curator_env):
|
||||
"""REPORT.md should be a valid markdown doc with the key sections visible."""
|
||||
curator = curator_env["curator"]
|
||||
start = datetime.now(timezone.utc)
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=start,
|
||||
elapsed_seconds=75.0,
|
||||
auto_counts={"checked": 10, "marked_stale": 2, "archived": 1, "reactivated": 0},
|
||||
auto_summary="2 marked stale, 1 archived",
|
||||
before_report=[{"name": "foo", "state": "active", "pinned": False}],
|
||||
before_names={"foo"},
|
||||
after_report=[{"name": "foo-umbrella", "state": "active", "pinned": False}],
|
||||
llm_meta=_make_llm_meta(
|
||||
final="Consolidated foo-like skills into foo-umbrella.",
|
||||
model="claude-opus-4.7",
|
||||
provider="openrouter",
|
||||
tool_calls=[
|
||||
# Evidence that `foo` was absorbed into `foo-umbrella`:
|
||||
# write_file under foo-umbrella referencing foo.
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "write_file",
|
||||
"name": "foo-umbrella",
|
||||
"file_path": "references/foo.md",
|
||||
"file_content": "# foo\nContent absorbed from the old foo skill.\n",
|
||||
}),
|
||||
},
|
||||
],
|
||||
),
|
||||
)
|
||||
md = (run_dir / "REPORT.md").read_text()
|
||||
|
||||
# Structural checks
|
||||
assert "# Curator run" in md
|
||||
assert "Auto-transitions" in md
|
||||
assert "LLM consolidation pass" in md
|
||||
assert "Recovery" in md
|
||||
|
||||
# The model / provider we passed in show up
|
||||
assert "claude-opus-4.7" in md
|
||||
assert "openrouter" in md
|
||||
|
||||
# The consolidated/added lists are present with clear language
|
||||
assert "Consolidated into umbrella skills" in md
|
||||
assert "`foo`" in md
|
||||
assert "merged into" in md
|
||||
assert "`foo-umbrella`" in md
|
||||
assert "New skills this run" in md
|
||||
|
||||
# The full LLM final response is included verbatim (no 240-char truncation)
|
||||
assert "Consolidated foo-like skills into foo-umbrella." in md
|
||||
|
||||
|
||||
def test_same_second_reruns_get_unique_dirs(curator_env):
|
||||
"""If the curator somehow runs twice in the same second, the second
|
||||
report still gets its own directory rather than overwriting the first."""
|
||||
curator = curator_env["curator"]
|
||||
start = datetime(2026, 4, 29, 5, 33, 34, tzinfo=timezone.utc)
|
||||
|
||||
kwargs = dict(
|
||||
started_at=start,
|
||||
elapsed_seconds=1.0,
|
||||
auto_counts={"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
||||
auto_summary="no changes",
|
||||
before_report=[],
|
||||
before_names=set(),
|
||||
after_report=[],
|
||||
llm_meta=_make_llm_meta(),
|
||||
)
|
||||
a = curator._write_run_report(**kwargs)
|
||||
b = curator._write_run_report(**kwargs)
|
||||
assert a != b
|
||||
assert a is not None and b is not None
|
||||
# Second dir has a numeric disambiguator suffix
|
||||
assert b.name.startswith(a.name)
|
||||
|
||||
|
||||
def test_report_captures_llm_error_and_continues(curator_env):
|
||||
"""If the LLM pass recorded an error, the report still writes and
|
||||
surfaces the error prominently."""
|
||||
curator = curator_env["curator"]
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=datetime.now(timezone.utc),
|
||||
elapsed_seconds=2.0,
|
||||
auto_counts={"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
||||
auto_summary="no changes",
|
||||
before_report=[],
|
||||
before_names=set(),
|
||||
after_report=[],
|
||||
llm_meta=_make_llm_meta(
|
||||
error="HTTP 400: No models provided",
|
||||
final="",
|
||||
summary="error",
|
||||
),
|
||||
)
|
||||
md = (run_dir / "REPORT.md").read_text()
|
||||
assert "HTTP 400" in md
|
||||
payload = json.loads((run_dir / "run.json").read_text())
|
||||
assert payload["llm_error"] == "HTTP 400: No models provided"
|
||||
|
||||
|
||||
def test_state_transitions_captured_in_report(curator_env):
|
||||
"""When a skill moves active → stale or stale → archived between
|
||||
before/after snapshots, the report records it."""
|
||||
curator = curator_env["curator"]
|
||||
start = datetime.now(timezone.utc)
|
||||
|
||||
before = [{"name": "getting-old", "state": "active", "pinned": False}]
|
||||
after = [{"name": "getting-old", "state": "stale", "pinned": False}]
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=start,
|
||||
elapsed_seconds=1.0,
|
||||
auto_counts={"checked": 1, "marked_stale": 1, "archived": 0, "reactivated": 0},
|
||||
auto_summary="1 marked stale",
|
||||
before_report=before,
|
||||
before_names={r["name"] for r in before},
|
||||
after_report=after,
|
||||
llm_meta=_make_llm_meta(),
|
||||
)
|
||||
payload = json.loads((run_dir / "run.json").read_text())
|
||||
assert payload["state_transitions"] == [
|
||||
{"name": "getting-old", "from": "active", "to": "stale"}
|
||||
]
|
||||
md = (run_dir / "REPORT.md").read_text()
|
||||
assert "State transitions" in md
|
||||
assert "getting-old" in md
|
||||
assert "active → stale" in md
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cron job skill reference rewriting (curator ↔ cron integration)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# When the curator consolidates skill X into umbrella Y during a run, any
|
||||
# cron job that listed X in its ``skills`` field would fail to load X at
|
||||
# run time — the scheduler logs a warning and skips it, so the scheduled
|
||||
# job runs without the instructions it was scheduled to follow. These
|
||||
# tests verify that _write_run_report calls into cron.jobs to repair
|
||||
# those references and records what it did in both run.json and
|
||||
# cron_rewrites.json.
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def curator_env_with_cron(curator_env, monkeypatch):
|
||||
"""Extend curator_env with an initialized + repointed cron.jobs module."""
|
||||
home = curator_env["home"]
|
||||
(home / "cron").mkdir(exist_ok=True)
|
||||
(home / "cron" / "output").mkdir(exist_ok=True)
|
||||
|
||||
import importlib
|
||||
import cron.jobs as jobs_mod
|
||||
importlib.reload(jobs_mod)
|
||||
monkeypatch.setattr(jobs_mod, "HERMES_DIR", home)
|
||||
monkeypatch.setattr(jobs_mod, "CRON_DIR", home / "cron")
|
||||
monkeypatch.setattr(jobs_mod, "JOBS_FILE", home / "cron" / "jobs.json")
|
||||
monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", home / "cron" / "output")
|
||||
|
||||
return {**curator_env, "jobs": jobs_mod}
|
||||
|
||||
|
||||
def test_curator_rewrites_cron_skills_when_skill_consolidated(curator_env_with_cron):
|
||||
"""A skill consolidated into an umbrella should be rewritten in any
|
||||
cron job's skills list; the rewrite should be visible in run.json
|
||||
and cron_rewrites.json."""
|
||||
curator = curator_env_with_cron["curator"]
|
||||
jobs = curator_env_with_cron["jobs"]
|
||||
|
||||
# Create a cron job that depends on a soon-to-be-consolidated skill
|
||||
job = jobs.create_job(
|
||||
prompt="",
|
||||
schedule="every 1h",
|
||||
skills=["foo"],
|
||||
name="foo-watcher",
|
||||
)
|
||||
|
||||
# Simulate a curator pass that consolidated `foo` → `foo-umbrella`
|
||||
before = [{"name": "foo", "state": "active", "pinned": False}]
|
||||
after = [{"name": "foo-umbrella", "state": "active", "pinned": False}]
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=datetime.now(timezone.utc),
|
||||
elapsed_seconds=3.0,
|
||||
auto_counts={"checked": 1, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
||||
auto_summary="no changes",
|
||||
before_report=before,
|
||||
before_names={"foo"},
|
||||
after_report=after,
|
||||
llm_meta=_make_llm_meta(
|
||||
final="Consolidated foo into foo-umbrella.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "write_file",
|
||||
"name": "foo-umbrella",
|
||||
"file_path": "references/foo.md",
|
||||
"file_content": "from foo",
|
||||
}),
|
||||
},
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
# Cron job is rewritten on disk
|
||||
loaded = jobs.get_job(job["id"])
|
||||
assert loaded["skills"] == ["foo-umbrella"]
|
||||
assert loaded["skill"] == "foo-umbrella"
|
||||
|
||||
# Rewrite is recorded in run.json
|
||||
payload = json.loads((run_dir / "run.json").read_text())
|
||||
assert payload["cron_rewrites"]["jobs_updated"] == 1
|
||||
assert payload["counts"]["cron_jobs_rewritten"] == 1
|
||||
rewrites = payload["cron_rewrites"]["rewrites"]
|
||||
assert len(rewrites) == 1
|
||||
assert rewrites[0]["mapped"] == {"foo": "foo-umbrella"}
|
||||
|
||||
# Separate cron_rewrites.json is written for convenience
|
||||
cron_file = run_dir / "cron_rewrites.json"
|
||||
assert cron_file.exists()
|
||||
detail = json.loads(cron_file.read_text())
|
||||
assert detail["jobs_updated"] == 1
|
||||
|
||||
# Markdown surfaces the change
|
||||
md = (run_dir / "REPORT.md").read_text()
|
||||
assert "Cron job skill references rewritten" in md
|
||||
assert "foo-watcher" in md
|
||||
assert "foo-umbrella" in md
|
||||
|
||||
|
||||
def test_curator_drops_pruned_skill_from_cron_job(curator_env_with_cron):
|
||||
"""A pruned (no-umbrella) skill should be dropped from the cron
|
||||
job's skill list entirely — there's no forwarding target."""
|
||||
curator = curator_env_with_cron["curator"]
|
||||
jobs = curator_env_with_cron["jobs"]
|
||||
|
||||
job = jobs.create_job(
|
||||
prompt="",
|
||||
schedule="every 1h",
|
||||
skills=["keep", "stale-one"],
|
||||
)
|
||||
|
||||
before = [{"name": "stale-one", "state": "active", "pinned": False}]
|
||||
after: list = [] # stale-one was archived with no target
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=datetime.now(timezone.utc),
|
||||
elapsed_seconds=1.0,
|
||||
auto_counts={"checked": 1, "marked_stale": 0, "archived": 1, "reactivated": 0},
|
||||
auto_summary="1 archived",
|
||||
before_report=before,
|
||||
before_names={"stale-one"},
|
||||
after_report=after,
|
||||
llm_meta=_make_llm_meta(), # no tool calls → classifier marks it pruned
|
||||
)
|
||||
|
||||
loaded = jobs.get_job(job["id"])
|
||||
assert loaded["skills"] == ["keep"]
|
||||
|
||||
payload = json.loads((run_dir / "run.json").read_text())
|
||||
assert payload["cron_rewrites"]["jobs_updated"] == 1
|
||||
rewrites = payload["cron_rewrites"]["rewrites"]
|
||||
assert rewrites[0]["dropped"] == ["stale-one"]
|
||||
|
||||
|
||||
def test_curator_report_has_no_cron_section_when_nothing_changes(curator_env_with_cron):
|
||||
"""When the curator run doesn't touch any skills, cron jobs are
|
||||
untouched and cron_rewrites.json is not even written."""
|
||||
curator = curator_env_with_cron["curator"]
|
||||
jobs = curator_env_with_cron["jobs"]
|
||||
|
||||
jobs.create_job(prompt="", schedule="every 1h", skills=["foo"])
|
||||
|
||||
run_dir = curator._write_run_report(
|
||||
started_at=datetime.now(timezone.utc),
|
||||
elapsed_seconds=1.0,
|
||||
auto_counts={"checked": 0, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
||||
auto_summary="no changes",
|
||||
before_report=[{"name": "foo", "state": "active", "pinned": False}],
|
||||
before_names={"foo"},
|
||||
after_report=[{"name": "foo", "state": "active", "pinned": False}],
|
||||
llm_meta=_make_llm_meta(),
|
||||
)
|
||||
|
||||
# No rewrites → no separate file, no section in md
|
||||
assert not (run_dir / "cron_rewrites.json").exists()
|
||||
md = (run_dir / "REPORT.md").read_text()
|
||||
assert "Cron job skill references rewritten" not in md
|
||||
|
||||
payload = json.loads((run_dir / "run.json").read_text())
|
||||
assert payload["cron_rewrites"]["jobs_updated"] == 0
|
||||
assert payload["counts"]["cron_jobs_rewritten"] == 0
|
||||
@@ -0,0 +1,93 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.agent_init import _merge_custom_provider_extra_body
|
||||
|
||||
|
||||
def test_custom_provider_extra_body_merges_into_request_overrides():
|
||||
agent = SimpleNamespace(
|
||||
provider="custom",
|
||||
model="google/gemma-4-31b-it",
|
||||
base_url="https://example.test/v1",
|
||||
request_overrides={"service_tier": "priority"},
|
||||
)
|
||||
|
||||
_merge_custom_provider_extra_body(
|
||||
agent,
|
||||
[
|
||||
{
|
||||
"name": "gemma",
|
||||
"base_url": "https://example.test/v1/",
|
||||
"model": "google/gemma-4-31b-it",
|
||||
"extra_body": {
|
||||
"enable_thinking": True,
|
||||
"reasoning_effort": "high",
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert agent.request_overrides == {
|
||||
"service_tier": "priority",
|
||||
"extra_body": {
|
||||
"enable_thinking": True,
|
||||
"reasoning_effort": "high",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_custom_provider_extra_body_preserves_caller_override():
|
||||
agent = SimpleNamespace(
|
||||
provider="custom",
|
||||
model="google/gemma-4-31b-it",
|
||||
base_url="https://example.test/v1",
|
||||
request_overrides={
|
||||
"extra_body": {
|
||||
"reasoning_effort": "low",
|
||||
"caller_only": True,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
_merge_custom_provider_extra_body(
|
||||
agent,
|
||||
[
|
||||
{
|
||||
"name": "gemma",
|
||||
"base_url": "https://example.test/v1",
|
||||
"model": "google/gemma-4-31b-it",
|
||||
"extra_body": {
|
||||
"enable_thinking": True,
|
||||
"reasoning_effort": "high",
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert agent.request_overrides["extra_body"] == {
|
||||
"enable_thinking": True,
|
||||
"reasoning_effort": "low",
|
||||
"caller_only": True,
|
||||
}
|
||||
|
||||
|
||||
def test_custom_provider_extra_body_ignores_other_custom_models():
|
||||
agent = SimpleNamespace(
|
||||
provider="custom",
|
||||
model="other-model",
|
||||
base_url="https://example.test/v1",
|
||||
request_overrides={},
|
||||
)
|
||||
|
||||
_merge_custom_provider_extra_body(
|
||||
agent,
|
||||
[
|
||||
{
|
||||
"name": "gemma",
|
||||
"base_url": "https://example.test/v1",
|
||||
"model": "google/gemma-4-31b-it",
|
||||
"extra_body": {"enable_thinking": True},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert agent.request_overrides == {}
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Regression guard: preserve thinking blocks on DeepSeek's /anthropic endpoint.
|
||||
|
||||
DeepSeek's ``api.deepseek.com/anthropic`` route speaks the Anthropic Messages
|
||||
protocol but, when thinking mode is enabled, requires ``thinking`` blocks from
|
||||
prior assistant turns to round-trip on subsequent requests. The generic
|
||||
third-party path strips them (signatures are Anthropic-proprietary and other
|
||||
proxies cannot validate them), so without a DeepSeek-specific carve-out the
|
||||
next tool-call turn fails with HTTP 400::
|
||||
|
||||
The content[].thinking in the thinking mode must be passed back to the
|
||||
API.
|
||||
|
||||
DeepSeek's compatibility matrix lists ``thinking`` as supported but
|
||||
``redacted_thinking`` and ``cache_control`` on thinking blocks as not
|
||||
supported. Handling is the same as Kimi's ``/coding`` endpoint: strip
|
||||
Anthropic-signed blocks (DeepSeek can't validate them) but preserve unsigned
|
||||
blocks that Hermes synthesises from ``reasoning_content``.
|
||||
|
||||
See hermes-agent#16748.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestDeepSeekAnthropicPreservesThinking:
|
||||
"""convert_messages_to_anthropic must replay DeepSeek thinking blocks."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://api.deepseek.com/anthropic",
|
||||
"https://api.deepseek.com/anthropic/",
|
||||
"https://api.deepseek.com/anthropic/v1",
|
||||
"https://API.DeepSeek.com/anthropic",
|
||||
],
|
||||
)
|
||||
def test_unsigned_thinking_block_survives_replay(self, base_url: str) -> None:
|
||||
"""Unsigned thinking (synthesised from reasoning_content) must be preserved."""
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "planning the tool call",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "skill_view", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
|
||||
]
|
||||
_system, converted = convert_messages_to_anthropic(
|
||||
messages, base_url=base_url
|
||||
)
|
||||
|
||||
assistant_msg = next(m for m in converted if m["role"] == "assistant")
|
||||
thinking_blocks = [
|
||||
b for b in assistant_msg["content"]
|
||||
if isinstance(b, dict) and b.get("type") == "thinking"
|
||||
]
|
||||
assert len(thinking_blocks) == 1, (
|
||||
f"DeepSeek /anthropic ({base_url}) must preserve unsigned thinking "
|
||||
"blocks synthesised from reasoning_content — upstream rejects "
|
||||
"replayed tool-call messages without them."
|
||||
)
|
||||
assert thinking_blocks[0]["thinking"] == "planning the tool call"
|
||||
# Synthesised block — never has a signature
|
||||
assert "signature" not in thinking_blocks[0]
|
||||
|
||||
def test_unsigned_thinking_preserved_on_non_latest_assistant_turn(self) -> None:
|
||||
"""DeepSeek validates history across every prior assistant turn, not just last."""
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "q1"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "r1",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
|
||||
{"role": "user", "content": "q2"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "r2",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "ok"},
|
||||
]
|
||||
_system, converted = convert_messages_to_anthropic(
|
||||
messages, base_url="https://api.deepseek.com/anthropic"
|
||||
)
|
||||
|
||||
assistants = [m for m in converted if m["role"] == "assistant"]
|
||||
assert len(assistants) == 2
|
||||
for assistant, expected in zip(assistants, ("r1", "r2")):
|
||||
thinking = [
|
||||
b for b in assistant["content"]
|
||||
if isinstance(b, dict) and b.get("type") == "thinking"
|
||||
]
|
||||
assert len(thinking) == 1
|
||||
assert thinking[0]["thinking"] == expected
|
||||
|
||||
def test_signed_anthropic_thinking_block_is_stripped(self) -> None:
|
||||
"""Anthropic-signed blocks (that leaked through) must still be stripped.
|
||||
|
||||
DeepSeek issues its own signatures and cannot validate Anthropic's —
|
||||
the strip-signed / keep-unsigned split matches the Kimi policy.
|
||||
"""
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "anthropic-signed payload",
|
||||
"signature": "anthropic-sig-xyz",
|
||||
},
|
||||
{"type": "text", "text": "hello"},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
_system, converted = convert_messages_to_anthropic(
|
||||
messages, base_url="https://api.deepseek.com/anthropic"
|
||||
)
|
||||
|
||||
assistant_msg = next(m for m in converted if m["role"] == "assistant")
|
||||
thinking_blocks = [
|
||||
b for b in assistant_msg["content"]
|
||||
if isinstance(b, dict) and b.get("type") == "thinking"
|
||||
]
|
||||
assert thinking_blocks == [], (
|
||||
"Signed Anthropic thinking blocks must be stripped on DeepSeek — "
|
||||
"DeepSeek cannot validate Anthropic-proprietary signatures."
|
||||
)
|
||||
|
||||
def test_cache_control_stripped_from_thinking_block(self) -> None:
|
||||
"""cache_control must still be stripped even when the block is preserved.
|
||||
|
||||
DeepSeek's compatibility matrix lists cache_control on thinking blocks
|
||||
as ignored — cache markers interfere with signature validation on
|
||||
upstreams that do check them, so Hermes strips them everywhere.
|
||||
"""
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "r1",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
|
||||
]
|
||||
# Inject cache_control on the synthesised thinking block after-the-fact
|
||||
# by running conversion once, mutating, then re-running would be
|
||||
# indirect. Instead check the simpler invariant: no thinking block in
|
||||
# the converted output carries cache_control.
|
||||
_system, converted = convert_messages_to_anthropic(
|
||||
messages, base_url="https://api.deepseek.com/anthropic"
|
||||
)
|
||||
for m in converted:
|
||||
if not isinstance(m.get("content"), list):
|
||||
continue
|
||||
for b in m["content"]:
|
||||
if isinstance(b, dict) and b.get("type") in {"thinking", "redacted_thinking"}:
|
||||
assert "cache_control" not in b
|
||||
|
||||
def test_openai_compat_deepseek_base_is_not_matched(self) -> None:
|
||||
"""The OpenAI-compatible ``api.deepseek.com`` base must NOT trigger the
|
||||
DeepSeek /anthropic branch — it never reaches this adapter, but the
|
||||
detector should still fail closed so an accidental misuse doesn't
|
||||
quietly send signed Anthropic blocks to an OpenAI endpoint.
|
||||
"""
|
||||
from agent.anthropic_adapter import _is_deepseek_anthropic_endpoint
|
||||
|
||||
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com") is False
|
||||
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com/v1") is False
|
||||
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com/anthropic") is True
|
||||
assert _is_deepseek_anthropic_endpoint("https://api.deepseek.com/anthropic/v1") is True
|
||||
|
||||
def test_non_deepseek_third_party_still_strips_all_thinking(self) -> None:
|
||||
"""MiniMax and other third-party Anthropic endpoints must keep the
|
||||
generic strip-all behaviour (they reject unsigned blocks outright).
|
||||
"""
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "r1",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
|
||||
]
|
||||
_system, converted = convert_messages_to_anthropic(
|
||||
messages, base_url="https://api.minimax.io/anthropic"
|
||||
)
|
||||
assistant_msg = next(m for m in converted if m["role"] == "assistant")
|
||||
thinking_blocks = [
|
||||
b for b in assistant_msg["content"]
|
||||
if isinstance(b, dict) and b.get("type") == "thinking"
|
||||
]
|
||||
assert thinking_blocks == [], (
|
||||
"Non-DeepSeek third-party endpoints must keep the generic "
|
||||
"strip-all-thinking behaviour — unsigned blocks get rejected."
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from run_agent import AIAgent
|
||||
|
||||
|
||||
def _agent_with_base_url(base_url: str) -> AIAgent:
|
||||
agent = object.__new__(AIAgent)
|
||||
agent.base_url = base_url
|
||||
return agent
|
||||
|
||||
|
||||
def test_direct_openai_url_requires_openai_host():
|
||||
agent = _agent_with_base_url("https://api.openai.com.example/v1")
|
||||
|
||||
assert agent._is_direct_openai_url() is False
|
||||
|
||||
|
||||
def test_direct_openai_url_ignores_path_segment_match():
|
||||
agent = _agent_with_base_url("https://proxy.example.test/api.openai.com/v1")
|
||||
|
||||
assert agent._is_direct_openai_url() is False
|
||||
|
||||
|
||||
def test_direct_openai_url_accepts_native_host():
|
||||
agent = _agent_with_base_url("https://api.openai.com/v1")
|
||||
|
||||
assert agent._is_direct_openai_url() is True
|
||||
@@ -0,0 +1,278 @@
|
||||
"""Tests for agent/display.py — build_tool_preview() and inline diff previews."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from agent.display import (
|
||||
build_tool_preview,
|
||||
capture_local_edit_snapshot,
|
||||
extract_edit_diff,
|
||||
get_cute_tool_message,
|
||||
set_tool_preview_max_len,
|
||||
_render_inline_unified_diff,
|
||||
_summarize_rendered_diff_sections,
|
||||
render_edit_diff_with_delta,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_tool_preview_max_len():
|
||||
set_tool_preview_max_len(0)
|
||||
yield
|
||||
set_tool_preview_max_len(0)
|
||||
|
||||
|
||||
class TestBuildToolPreview:
|
||||
"""Tests for build_tool_preview defensive handling and normal operation."""
|
||||
|
||||
def test_none_args_returns_none(self):
|
||||
"""PR #453: None args should not crash, should return None."""
|
||||
assert build_tool_preview("terminal", None) is None
|
||||
|
||||
def test_empty_dict_returns_none(self):
|
||||
"""Empty dict has no keys to preview."""
|
||||
assert build_tool_preview("terminal", {}) is None
|
||||
|
||||
def test_known_tool_with_primary_arg(self):
|
||||
"""Known tool with its primary arg should return a preview string."""
|
||||
result = build_tool_preview("terminal", {"command": "ls -la"})
|
||||
assert result is not None
|
||||
assert "ls -la" in result
|
||||
|
||||
def test_web_search_preview(self):
|
||||
result = build_tool_preview("web_search", {"query": "hello world"})
|
||||
assert result is not None
|
||||
assert "hello world" in result
|
||||
|
||||
def test_read_file_preview(self):
|
||||
result = build_tool_preview("read_file", {"path": "/tmp/test.py", "offset": 1})
|
||||
assert result is not None
|
||||
assert "/tmp/test.py" in result
|
||||
|
||||
def test_unknown_tool_with_fallback_key(self):
|
||||
"""Unknown tool but with a recognized fallback key should still preview."""
|
||||
result = build_tool_preview("custom_tool", {"query": "test query"})
|
||||
assert result is not None
|
||||
assert "test query" in result
|
||||
|
||||
def test_unknown_tool_no_matching_key(self):
|
||||
"""Unknown tool with no recognized keys should return None."""
|
||||
result = build_tool_preview("custom_tool", {"foo": "bar"})
|
||||
assert result is None
|
||||
|
||||
def test_long_value_truncated(self):
|
||||
"""Preview should truncate long values."""
|
||||
long_cmd = "a" * 100
|
||||
result = build_tool_preview("terminal", {"command": long_cmd}, max_len=40)
|
||||
assert result is not None
|
||||
assert len(result) <= 43 # max_len + "..."
|
||||
|
||||
def test_process_tool_with_none_args(self):
|
||||
"""Process tool special case should also handle None args."""
|
||||
assert build_tool_preview("process", None) is None
|
||||
|
||||
def test_process_tool_normal(self):
|
||||
result = build_tool_preview("process", {"action": "poll", "session_id": "abc123"})
|
||||
assert result is not None
|
||||
assert "poll" in result
|
||||
|
||||
def test_todo_tool_read(self):
|
||||
result = build_tool_preview("todo", {"merge": False})
|
||||
assert result is not None
|
||||
assert "reading" in result
|
||||
|
||||
def test_todo_tool_with_todos(self):
|
||||
result = build_tool_preview("todo", {"todos": [{"id": "1", "content": "test", "status": "pending"}]})
|
||||
assert result is not None
|
||||
assert "1 task" in result
|
||||
|
||||
def test_memory_tool_add(self):
|
||||
result = build_tool_preview("memory", {"action": "add", "target": "user", "content": "test note"})
|
||||
assert result is not None
|
||||
assert "user" in result
|
||||
|
||||
def test_memory_replace_missing_old_text_marked(self):
|
||||
# Avoid empty quotes "" in the preview when old_text is missing/None.
|
||||
result = build_tool_preview("memory", {"action": "replace", "target": "memory"})
|
||||
assert result == '~memory: "<missing old_text>"'
|
||||
result = build_tool_preview("memory", {"action": "remove", "target": "memory", "old_text": None})
|
||||
assert result == '-memory: "<missing old_text>"'
|
||||
|
||||
def test_session_search_preview(self):
|
||||
result = build_tool_preview("session_search", {"query": "find something"})
|
||||
assert result is not None
|
||||
assert "find something" in result
|
||||
|
||||
def test_false_like_args_zero(self):
|
||||
"""Non-dict falsy values should return None, not crash."""
|
||||
assert build_tool_preview("terminal", 0) is None
|
||||
assert build_tool_preview("terminal", "") is None
|
||||
assert build_tool_preview("terminal", []) is None
|
||||
|
||||
|
||||
class TestCuteToolMessagePreviewLength:
|
||||
def test_terminal_preview_unlimited_when_config_is_zero(self):
|
||||
set_tool_preview_max_len(0)
|
||||
command = "curl -s http://localhost:9222/json/list | jq -r '.[] | select(.type==\"page\")' | head -5"
|
||||
|
||||
line = get_cute_tool_message("terminal", {"command": command}, 0.1)
|
||||
|
||||
assert command in line
|
||||
assert "..." not in line
|
||||
|
||||
def test_terminal_preview_uses_positive_configured_limit(self):
|
||||
set_tool_preview_max_len(80)
|
||||
command = "curl -s http://localhost:9222/json/list | jq -r '.[] | select(.type==\"page\")' | head -5"
|
||||
|
||||
line = get_cute_tool_message("terminal", {"command": command}, 0.1)
|
||||
|
||||
assert command[:77] in line
|
||||
assert "..." in line
|
||||
assert "head -5" not in line
|
||||
|
||||
def test_search_files_preview_uses_positive_configured_limit_not_default(self):
|
||||
set_tool_preview_max_len(80)
|
||||
pattern = "function.formatToolCall.context.preview.compactPreview.maxLength.truncate"
|
||||
|
||||
line = get_cute_tool_message("search_files", {"pattern": pattern}, 0.1)
|
||||
|
||||
assert pattern in line
|
||||
assert "..." not in line
|
||||
|
||||
def test_path_preview_uses_positive_configured_limit_not_default(self):
|
||||
set_tool_preview_max_len(80)
|
||||
path = "/tmp/hermes-test-preview-length/deeply/nested/path/test-output.txt"
|
||||
|
||||
line = get_cute_tool_message("read_file", {"path": path}, 0.1)
|
||||
|
||||
assert path in line
|
||||
assert "..." not in line
|
||||
|
||||
def test_write_file_lint_error_result_is_not_marked_failed(self):
|
||||
result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
|
||||
line = get_cute_tool_message("write_file", {"path": "/tmp/a.py"}, 0.1, result=result)
|
||||
|
||||
assert "[error]" not in line
|
||||
|
||||
def test_patch_lsp_diagnostics_result_is_not_marked_failed(self):
|
||||
result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
line = get_cute_tool_message("patch", {"path": "/tmp/a.py"}, 0.1, result=result)
|
||||
|
||||
assert "[error]" not in line
|
||||
|
||||
|
||||
class TestEditDiffPreview:
|
||||
def test_extract_edit_diff_for_patch(self):
|
||||
diff = extract_edit_diff("patch", '{"success": true, "diff": "--- a/x\\n+++ b/x\\n"}')
|
||||
assert diff is not None
|
||||
assert "+++ b/x" in diff
|
||||
|
||||
def test_render_inline_unified_diff_colors_added_and_removed_lines(self):
|
||||
rendered = _render_inline_unified_diff(
|
||||
"--- a/cli.py\n"
|
||||
"+++ b/cli.py\n"
|
||||
"@@ -1,2 +1,2 @@\n"
|
||||
"-old line\n"
|
||||
"+new line\n"
|
||||
" context\n"
|
||||
)
|
||||
|
||||
assert "a/cli.py" in rendered[0]
|
||||
assert "b/cli.py" in rendered[0]
|
||||
assert any("old line" in line for line in rendered)
|
||||
assert any("new line" in line for line in rendered)
|
||||
assert any("48;2;" in line for line in rendered)
|
||||
|
||||
def test_extract_edit_diff_ignores_non_edit_tools(self):
|
||||
assert extract_edit_diff("web_search", '{"diff": "--- a\\n+++ b\\n"}') is None
|
||||
|
||||
def test_extract_edit_diff_uses_local_snapshot_for_write_file(self, tmp_path):
|
||||
target = tmp_path / "note.txt"
|
||||
target.write_text("old\n", encoding="utf-8")
|
||||
|
||||
snapshot = capture_local_edit_snapshot("write_file", {"path": str(target)})
|
||||
|
||||
target.write_text("new\n", encoding="utf-8")
|
||||
|
||||
diff = extract_edit_diff(
|
||||
"write_file",
|
||||
'{"bytes_written": 4}',
|
||||
function_args={"path": str(target)},
|
||||
snapshot=snapshot,
|
||||
)
|
||||
|
||||
assert diff is not None
|
||||
assert "--- a/" in diff
|
||||
assert "+++ b/" in diff
|
||||
assert "-old" in diff
|
||||
assert "+new" in diff
|
||||
|
||||
def test_render_edit_diff_with_delta_invokes_printer(self):
|
||||
printer = MagicMock()
|
||||
|
||||
rendered = render_edit_diff_with_delta(
|
||||
"patch",
|
||||
'{"diff": "--- a/x\\n+++ b/x\\n@@ -1 +1 @@\\n-old\\n+new\\n"}',
|
||||
print_fn=printer,
|
||||
)
|
||||
|
||||
assert rendered is True
|
||||
assert printer.call_count >= 2
|
||||
calls = [call.args[0] for call in printer.call_args_list]
|
||||
assert any("a/x" in line and "b/x" in line for line in calls)
|
||||
assert any("old" in line for line in calls)
|
||||
assert any("new" in line for line in calls)
|
||||
|
||||
def test_render_edit_diff_with_delta_skips_without_diff(self):
|
||||
rendered = render_edit_diff_with_delta(
|
||||
"patch",
|
||||
'{"success": true}',
|
||||
)
|
||||
|
||||
assert rendered is False
|
||||
|
||||
def test_render_edit_diff_with_delta_handles_renderer_errors(self, monkeypatch):
|
||||
printer = MagicMock()
|
||||
|
||||
monkeypatch.setattr("agent.display._summarize_rendered_diff_sections", MagicMock(side_effect=RuntimeError("boom")))
|
||||
|
||||
rendered = render_edit_diff_with_delta(
|
||||
"patch",
|
||||
'{"diff": "--- a/x\\n+++ b/x\\n"}',
|
||||
print_fn=printer,
|
||||
)
|
||||
|
||||
assert rendered is False
|
||||
assert printer.call_count == 0
|
||||
|
||||
def test_summarize_rendered_diff_sections_truncates_large_diff(self):
|
||||
diff = "--- a/x.py\n+++ b/x.py\n" + "".join(f"+line{i}\n" for i in range(120))
|
||||
|
||||
rendered = _summarize_rendered_diff_sections(diff, max_lines=20)
|
||||
|
||||
assert len(rendered) == 21
|
||||
assert "omitted" in rendered[-1]
|
||||
|
||||
def test_summarize_rendered_diff_sections_limits_file_count(self):
|
||||
diff = "".join(
|
||||
f"--- a/file{i}.py\n+++ b/file{i}.py\n+line{i}\n"
|
||||
for i in range(8)
|
||||
)
|
||||
|
||||
rendered = _summarize_rendered_diff_sections(diff, max_files=3, max_lines=50)
|
||||
|
||||
assert any("a/file0.py" in line for line in rendered)
|
||||
assert any("a/file1.py" in line for line in rendered)
|
||||
assert any("a/file2.py" in line for line in rendered)
|
||||
assert not any("a/file7.py" in line for line in rendered)
|
||||
assert "additional file" in rendered[-1]
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Tests for get_tool_emoji in agent/display.py — skin + registry integration."""
|
||||
|
||||
from unittest.mock import patch as mock_patch, MagicMock
|
||||
|
||||
from agent.display import get_tool_emoji
|
||||
|
||||
|
||||
class TestGetToolEmoji:
|
||||
"""Verify the skin → registry → fallback resolution chain."""
|
||||
|
||||
def test_returns_registry_emoji_when_no_skin(self):
|
||||
"""Registry-registered emoji is used when no skin is active."""
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get_emoji.return_value = "🎨"
|
||||
with mock_patch("agent.display._get_skin", return_value=None), \
|
||||
mock_patch("agent.display.registry", mock_registry, create=True):
|
||||
# Need to patch the import inside get_tool_emoji
|
||||
pass
|
||||
# Direct test: patch the lazy import path
|
||||
with mock_patch("agent.display._get_skin", return_value=None):
|
||||
# get_tool_emoji will try to import registry — mock that
|
||||
mock_reg = MagicMock()
|
||||
mock_reg.get_emoji.return_value = "📖"
|
||||
with mock_patch.dict("sys.modules", {}):
|
||||
import sys
|
||||
# Patch tools.registry module
|
||||
mock_module = MagicMock()
|
||||
mock_module.registry = mock_reg
|
||||
with mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
|
||||
result = get_tool_emoji("read_file")
|
||||
assert result == "📖"
|
||||
|
||||
def test_skin_override_takes_precedence(self):
|
||||
"""Skin tool_emojis override registry defaults."""
|
||||
skin = MagicMock()
|
||||
skin.tool_emojis = {"terminal": "⚔"}
|
||||
with mock_patch("agent.display._get_skin", return_value=skin):
|
||||
result = get_tool_emoji("terminal")
|
||||
assert result == "⚔"
|
||||
|
||||
def test_skin_empty_dict_falls_through(self):
|
||||
"""Empty skin tool_emojis falls through to registry."""
|
||||
skin = MagicMock()
|
||||
skin.tool_emojis = {}
|
||||
mock_reg = MagicMock()
|
||||
mock_reg.get_emoji.return_value = "💻"
|
||||
import sys
|
||||
mock_module = MagicMock()
|
||||
mock_module.registry = mock_reg
|
||||
with mock_patch("agent.display._get_skin", return_value=skin), \
|
||||
mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
|
||||
result = get_tool_emoji("terminal")
|
||||
assert result == "💻"
|
||||
|
||||
def test_fallback_default(self):
|
||||
"""When neither skin nor registry has an emoji, use the default."""
|
||||
skin = MagicMock()
|
||||
skin.tool_emojis = {}
|
||||
mock_reg = MagicMock()
|
||||
mock_reg.get_emoji.return_value = ""
|
||||
import sys
|
||||
mock_module = MagicMock()
|
||||
mock_module.registry = mock_reg
|
||||
with mock_patch("agent.display._get_skin", return_value=skin), \
|
||||
mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
|
||||
result = get_tool_emoji("unknown_tool")
|
||||
assert result == "⚡"
|
||||
|
||||
def test_custom_default(self):
|
||||
"""Custom default is returned when nothing matches."""
|
||||
with mock_patch("agent.display._get_skin", return_value=None):
|
||||
mock_reg = MagicMock()
|
||||
mock_reg.get_emoji.return_value = ""
|
||||
import sys
|
||||
mock_module = MagicMock()
|
||||
mock_module.registry = mock_reg
|
||||
with mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
|
||||
result = get_tool_emoji("x", default="⚙️")
|
||||
assert result == "⚙️"
|
||||
|
||||
def test_skin_override_only_for_matching_tool(self):
|
||||
"""Skin override for one tool doesn't affect others."""
|
||||
skin = MagicMock()
|
||||
skin.tool_emojis = {"terminal": "⚔"}
|
||||
mock_reg = MagicMock()
|
||||
mock_reg.get_emoji.return_value = "🔍"
|
||||
import sys
|
||||
mock_module = MagicMock()
|
||||
mock_module.registry = mock_reg
|
||||
with mock_patch("agent.display._get_skin", return_value=skin), \
|
||||
mock_patch.dict(sys.modules, {"tools.registry": mock_module}):
|
||||
assert get_tool_emoji("terminal") == "⚔" # skin override
|
||||
assert get_tool_emoji("web_search") == "🔍" # registry fallback
|
||||
|
||||
|
||||
class TestSkinConfigToolEmojis:
|
||||
"""Verify SkinConfig handles tool_emojis field correctly."""
|
||||
|
||||
def test_skin_config_has_tool_emojis_field(self):
|
||||
from hermes_cli.skin_engine import SkinConfig
|
||||
skin = SkinConfig(name="test")
|
||||
assert skin.tool_emojis == {}
|
||||
|
||||
def test_skin_config_accepts_tool_emojis(self):
|
||||
from hermes_cli.skin_engine import SkinConfig
|
||||
emojis = {"terminal": "⚔", "web_search": "🔮"}
|
||||
skin = SkinConfig(name="test", tool_emojis=emojis)
|
||||
assert skin.tool_emojis == emojis
|
||||
|
||||
def test_build_skin_config_includes_tool_emojis(self):
|
||||
from hermes_cli.skin_engine import _build_skin_config
|
||||
data = {
|
||||
"name": "custom",
|
||||
"tool_emojis": {"terminal": "🗡️", "patch": "⚒️"},
|
||||
}
|
||||
skin = _build_skin_config(data)
|
||||
assert skin.tool_emojis == {"terminal": "🗡️", "patch": "⚒️"}
|
||||
|
||||
def test_build_skin_config_empty_tool_emojis_default(self):
|
||||
from hermes_cli.skin_engine import _build_skin_config
|
||||
data = {"name": "minimal"}
|
||||
skin = _build_skin_config(data)
|
||||
assert skin.tool_emojis == {}
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Tests for get_cute_tool_message todo progress display.
|
||||
|
||||
Verifies the completion status rendering (done/total ✓) on all three
|
||||
todo tool call paths: read, create (merge=False), update (merge=True).
|
||||
"""
|
||||
|
||||
import json
|
||||
from agent.display import get_cute_tool_message
|
||||
|
||||
|
||||
def _todo_result(total: int, completed: int) -> str:
|
||||
"""Build a fake todo_tool return value."""
|
||||
return json.dumps({
|
||||
"todos": [],
|
||||
"summary": {
|
||||
"total": total,
|
||||
"pending": total - completed,
|
||||
"in_progress": 0,
|
||||
"completed": completed,
|
||||
"cancelled": 0,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
class TestTodoRead:
|
||||
"""get_cute_tool_message(…, result=…) when todos_arg is None (read path)."""
|
||||
|
||||
def test_read_no_result(self):
|
||||
msg = get_cute_tool_message("todo", {}, 0.5)
|
||||
assert "reading tasks" in msg
|
||||
assert "0.5s" in msg
|
||||
|
||||
def test_read_with_progress(self):
|
||||
msg = get_cute_tool_message("todo", {}, 0.5,
|
||||
result=_todo_result(4, 2))
|
||||
assert "2/4" in msg
|
||||
assert "task(s)" in msg
|
||||
|
||||
def test_read_all_done(self):
|
||||
msg = get_cute_tool_message("todo", {}, 0.5,
|
||||
result=_todo_result(4, 4))
|
||||
assert "4/4" in msg
|
||||
assert "task(s)" in msg
|
||||
|
||||
def test_read_zero_total(self):
|
||||
"""Edge case: empty todo list returns summary with total=0."""
|
||||
msg = get_cute_tool_message("todo", {}, 0.5,
|
||||
result=_todo_result(0, 0))
|
||||
assert "reading tasks" in msg
|
||||
|
||||
def test_read_invalid_result_fallback(self):
|
||||
"""Garbage result should not crash; fall back to reading tasks."""
|
||||
msg = get_cute_tool_message("todo", {}, 0.5, result="not json")
|
||||
assert "reading tasks" in msg
|
||||
|
||||
def test_read_result_missing_summary(self):
|
||||
msg = get_cute_tool_message("todo", {}, 0.5,
|
||||
result='{"todos": []}')
|
||||
assert "reading tasks" in msg
|
||||
|
||||
|
||||
class TestTodoCreate:
|
||||
"""get_cute_tool_message when merge=False (new plan creation)."""
|
||||
|
||||
def test_create_default(self):
|
||||
"""Brand-new plan: all pending, no result — plain count."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [
|
||||
{"id": "a", "content": "x", "status": "pending"},
|
||||
]}, 0.3)
|
||||
assert "1 task(s)" in msg
|
||||
assert "0.3s" in msg
|
||||
assert "/" not in msg # no progress fraction
|
||||
|
||||
def test_create_multiple(self):
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [
|
||||
{"id": "a", "content": "x", "status": "pending"},
|
||||
{"id": "b", "content": "y", "status": "pending"},
|
||||
{"id": "c", "content": "z", "status": "pending"},
|
||||
]}, 0.2)
|
||||
assert "3 task(s)" in msg
|
||||
|
||||
def test_create_with_result_shows_progress_when_done(self):
|
||||
"""Even on create, if result has completed tasks show it."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "content": "x", "status": "completed"}]},
|
||||
0.4,
|
||||
result=_todo_result(1, 1))
|
||||
assert "1/1" in msg
|
||||
assert "task(s)" in msg
|
||||
|
||||
def test_create_with_result_zero_done(self):
|
||||
"""New plan with 0 done — plain count, no progress fraction."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [
|
||||
{"id": "a", "content": "x", "status": "pending"},
|
||||
{"id": "b", "content": "y", "status": "pending"},
|
||||
]},
|
||||
0.3,
|
||||
result=_todo_result(2, 0))
|
||||
assert "2 task(s)" in msg
|
||||
assert "/" not in msg
|
||||
|
||||
|
||||
class TestTodoUpdate:
|
||||
"""get_cute_tool_message when merge=True (incremental update)."""
|
||||
|
||||
def test_update_no_result(self):
|
||||
"""No result available — plain update N task(s)."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "status": "completed"}],
|
||||
"merge": True}, 0.5)
|
||||
assert "update 1 task(s)" in msg
|
||||
|
||||
def test_update_partial_progress(self):
|
||||
"""1/4 tasks completed — show fraction with checkmark."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "status": "completed"}],
|
||||
"merge": True},
|
||||
0.5,
|
||||
result=_todo_result(4, 1))
|
||||
assert "update" in msg
|
||||
assert "1/4" in msg
|
||||
assert "✓" in msg
|
||||
|
||||
def test_update_halfway(self):
|
||||
"""2/4 — midpoint progress."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "b", "status": "in_progress"}],
|
||||
"merge": True},
|
||||
0.7,
|
||||
result=_todo_result(4, 2))
|
||||
assert "2/4" in msg
|
||||
assert "✓" in msg
|
||||
|
||||
def test_update_all_completed(self):
|
||||
"""4/4 — full checkmark."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "d", "status": "completed"}],
|
||||
"merge": True},
|
||||
0.2,
|
||||
result=_todo_result(4, 4))
|
||||
assert "4/4" in msg
|
||||
assert "✓" in msg
|
||||
|
||||
def test_update_zero_done(self):
|
||||
"""No completed tasks yet — plain update N task(s)."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "status": "pending"}],
|
||||
"merge": True},
|
||||
0.3,
|
||||
result=_todo_result(3, 0))
|
||||
assert "update 1 task(s)" in msg
|
||||
assert "✓" not in msg
|
||||
assert "/" not in msg # no progress fraction when done=0
|
||||
|
||||
def test_update_invalid_result_fallback(self):
|
||||
"""Bad JSON result — fall back to plain update N task(s)."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "status": "completed"}],
|
||||
"merge": True},
|
||||
0.6,
|
||||
result="{broken")
|
||||
assert "update 1 task(s)" in msg
|
||||
assert "✓" not in msg
|
||||
|
||||
def test_update_result_missing_summary(self):
|
||||
"""Result no summary key — fall back to plain update."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "status": "completed"}],
|
||||
"merge": True},
|
||||
0.4,
|
||||
result='{"todos": []}')
|
||||
assert "update 1 task(s)" in msg
|
||||
assert "✓" not in msg
|
||||
|
||||
def test_update_total_not_in_summary(self):
|
||||
"""Result summary missing total key."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "status": "completed"}],
|
||||
"merge": True},
|
||||
0.3,
|
||||
result=json.dumps({"summary": {"completed": 2}}))
|
||||
assert "update 1 task(s)" in msg
|
||||
assert "✓" not in msg
|
||||
|
||||
def test_update_multiple_tasks_in_line(self):
|
||||
"""Update line with several tasks in the update request."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [
|
||||
{"id": "a", "status": "completed"},
|
||||
{"id": "b", "status": "in_progress"},
|
||||
], "merge": True},
|
||||
0.5,
|
||||
result=_todo_result(5, 3))
|
||||
assert "update" in msg
|
||||
assert "3/5" in msg
|
||||
assert "✓" in msg
|
||||
|
||||
|
||||
class TestTodoEdgeCases:
|
||||
"""Boundary cases that should not crash."""
|
||||
|
||||
def test_merge_default_value(self):
|
||||
"""merge defaults to False in function signature, should be False when absent."""
|
||||
msg = get_cute_tool_message("todo",
|
||||
{"todos": [{"id": "a", "content": "x", "status": "pending"}]},
|
||||
1.0)
|
||||
assert "1 task(s)" in msg
|
||||
|
||||
def test_duration_formatting(self):
|
||||
"""Duration formatting works correctly."""
|
||||
msg = get_cute_tool_message("todo", {}, 0.123)
|
||||
assert "0.1s" in msg
|
||||
|
||||
msg = get_cute_tool_message("todo", {}, 1.0)
|
||||
assert "1.0s" in msg
|
||||
|
||||
msg = get_cute_tool_message("todo", {}, 123.456)
|
||||
assert "123.5s" in msg
|
||||
|
||||
def test_large_task_count(self):
|
||||
"""Many tasks should not break formatting."""
|
||||
many = [{"id": str(i), "content": "x", "status": "pending"} for i in range(50)]
|
||||
msg = get_cute_tool_message("todo", {"todos": many}, 0.5)
|
||||
assert "50 task(s)" in msg
|
||||
|
||||
def test_read_with_no_args_and_no_result(self):
|
||||
"""Completely empty call."""
|
||||
msg = get_cute_tool_message("todo", {}, 0.0)
|
||||
assert "reading tasks" in msg
|
||||
|
||||
|
||||
class TestTodoSkinIntegration:
|
||||
"""Verify the skin prefix is applied to todo messages too.
|
||||
This uses the same pattern as test_skin_engine test_tool_message_uses_skin_prefix.
|
||||
"""
|
||||
|
||||
def test_default_skin_prefix(self):
|
||||
msg = get_cute_tool_message("todo", {}, 0.5)
|
||||
assert msg.startswith("┊")
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Tests for _detect_tool_failure + _trim_error + get_cute_tool_message
|
||||
inline failure suffix rendering.
|
||||
|
||||
Covers the user-visible promise: when a tool fails, the CLI shows a short,
|
||||
specific reason in square brackets at the end of the completion line —
|
||||
not a generic "[error]".
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from agent.display import (
|
||||
_detect_tool_failure,
|
||||
_trim_error,
|
||||
_ERROR_SUFFIX_MAX_LEN,
|
||||
get_cute_tool_message,
|
||||
)
|
||||
|
||||
|
||||
class TestTrimError:
|
||||
"""The helper that shrinks an error message for inline display."""
|
||||
|
||||
def test_short_message_unchanged(self):
|
||||
assert _trim_error("nope") == "nope"
|
||||
|
||||
def test_whitespace_stripped(self):
|
||||
assert _trim_error(" bad input ") == "bad input"
|
||||
|
||||
def test_long_message_truncated_to_cap(self):
|
||||
msg = "x" * 200
|
||||
trimmed = _trim_error(msg)
|
||||
assert len(trimmed) <= _ERROR_SUFFIX_MAX_LEN
|
||||
assert trimmed.endswith("...")
|
||||
|
||||
def test_file_not_found_path_collapsed_to_filename(self):
|
||||
long_path = "File not found: /home/teknium/.hermes/hermes-agent/very/deep/path/foo.py"
|
||||
assert _trim_error(long_path) == "File not found: foo.py"
|
||||
|
||||
def test_file_not_found_already_short_unchanged(self):
|
||||
assert _trim_error("File not found: foo.py") == "File not found: foo.py"
|
||||
|
||||
def test_file_not_found_relative_path_unchanged(self):
|
||||
# Without a slash there's no path to trim.
|
||||
assert _trim_error("File not found: foo.py") == "File not found: foo.py"
|
||||
|
||||
|
||||
class TestDetectToolFailureTerminal:
|
||||
"""terminal: non-zero exit_code is the canonical failure signal."""
|
||||
|
||||
def test_success_returns_no_suffix(self):
|
||||
result = json.dumps({"output": "ok\n", "exit_code": 0})
|
||||
assert _detect_tool_failure("terminal", result) == (False, "")
|
||||
|
||||
def test_nonzero_exit_with_no_error_shows_exit_code(self):
|
||||
result = json.dumps({"output": "", "exit_code": 1})
|
||||
is_failure, suffix = _detect_tool_failure("terminal", result)
|
||||
assert is_failure is True
|
||||
assert suffix == " [exit 1]"
|
||||
|
||||
def test_nonzero_exit_with_error_shows_message(self):
|
||||
result = json.dumps({
|
||||
"output": "",
|
||||
"exit_code": 127,
|
||||
"error": "ls: cannot access 'foo': No such file or directory",
|
||||
})
|
||||
is_failure, suffix = _detect_tool_failure("terminal", result)
|
||||
assert is_failure is True
|
||||
assert "cannot access" in suffix
|
||||
# Trimmed to the cap, in brackets
|
||||
assert suffix.startswith(" [")
|
||||
assert suffix.endswith("]")
|
||||
|
||||
def test_malformed_json_returns_no_suffix(self):
|
||||
# Terminal is special: only exit_code matters. Malformed JSON should
|
||||
# not crash and should not be flagged as failure.
|
||||
assert _detect_tool_failure("terminal", "not json") == (False, "")
|
||||
|
||||
def test_none_result_returns_no_suffix(self):
|
||||
assert _detect_tool_failure("terminal", None) == (False, "")
|
||||
|
||||
|
||||
class TestDetectToolFailureMemory:
|
||||
"""memory: 'full' is distinct from real errors."""
|
||||
|
||||
def test_memory_full_returns_full_suffix(self):
|
||||
result = json.dumps({"success": False, "error": "would exceed the limit"})
|
||||
assert _detect_tool_failure("memory", result) == (True, " [full]")
|
||||
|
||||
def test_memory_other_error_returns_specific_message(self):
|
||||
# An error that's NOT a "full" overflow falls through to the
|
||||
# structured-error path and surfaces the actual message.
|
||||
result = json.dumps({"success": False, "error": "invalid action: zap"})
|
||||
is_failure, suffix = _detect_tool_failure("memory", result)
|
||||
assert is_failure is True
|
||||
assert "invalid action" in suffix
|
||||
|
||||
|
||||
class TestDetectToolFailureStructured:
|
||||
"""Generic path: any tool that returns {"error": ...} JSON."""
|
||||
|
||||
def test_read_file_error_surfaced(self):
|
||||
result = json.dumps({
|
||||
"path": "/nope/missing.py",
|
||||
"success": False,
|
||||
"error": "File not found: /nope/missing.py",
|
||||
})
|
||||
is_failure, suffix = _detect_tool_failure("read_file", result)
|
||||
assert is_failure is True
|
||||
# _trim_error reduces the path to the basename.
|
||||
assert suffix == " [File not found: missing.py]"
|
||||
|
||||
def test_error_without_success_key_still_flagged(self):
|
||||
# Some tools return {"error": "..."} with no explicit success flag.
|
||||
result = json.dumps({"error": "remote unavailable"})
|
||||
is_failure, suffix = _detect_tool_failure("web_search", result)
|
||||
assert is_failure is True
|
||||
assert suffix == " [remote unavailable]"
|
||||
|
||||
def test_message_field_only_with_success_false_flagged(self):
|
||||
# When success is False and only 'message' is set, surface it.
|
||||
result = json.dumps({"success": False, "message": "rate limited"})
|
||||
is_failure, suffix = _detect_tool_failure("web_search", result)
|
||||
assert is_failure is True
|
||||
assert "rate limited" in suffix
|
||||
|
||||
def test_successful_result_not_flagged(self):
|
||||
result = json.dumps({"success": True, "data": "hello"})
|
||||
assert _detect_tool_failure("web_search", result) == (False, "")
|
||||
|
||||
def test_dict_without_error_or_success_uses_generic_heuristic(self):
|
||||
# Plain successful dict — should pass through the generic
|
||||
# heuristic which only fires on the string "Error" / '"error"' / etc.
|
||||
result = json.dumps({"data": "hello"})
|
||||
is_failure, _ = _detect_tool_failure("web_search", result)
|
||||
assert is_failure is False
|
||||
|
||||
|
||||
class TestGetCuteToolMessageFailureSuffix:
|
||||
"""End-to-end: failure suffix is appended by get_cute_tool_message."""
|
||||
|
||||
def test_read_file_failure_suffix_appended(self):
|
||||
fail = json.dumps({
|
||||
"path": "/etc/missing",
|
||||
"success": False,
|
||||
"error": "File not found: /etc/missing",
|
||||
})
|
||||
line = get_cute_tool_message("read_file", {"path": "/etc/missing"}, 0.1, result=fail)
|
||||
assert "[File not found: missing]" in line
|
||||
|
||||
def test_terminal_exit_only_suffix(self):
|
||||
fail = json.dumps({"output": "", "exit_code": 2})
|
||||
line = get_cute_tool_message("terminal", {"command": "false"}, 0.1, result=fail)
|
||||
assert "[exit 2]" in line
|
||||
|
||||
def test_terminal_with_stderr_uses_message(self):
|
||||
fail = json.dumps({
|
||||
"output": "",
|
||||
"exit_code": 127,
|
||||
"error": "command not found: notathing",
|
||||
})
|
||||
line = get_cute_tool_message("terminal", {"command": "notathing"}, 0.1, result=fail)
|
||||
assert "command not found" in line
|
||||
# No '[exit 127]' tag when we have a specific message
|
||||
assert "exit 127" not in line
|
||||
|
||||
def test_memory_full_suffix(self):
|
||||
fail = json.dumps({"success": False, "error": "would exceed the limit"})
|
||||
line = get_cute_tool_message(
|
||||
"memory",
|
||||
{"action": "add", "target": "memory", "content": "x"},
|
||||
0.05,
|
||||
result=fail,
|
||||
)
|
||||
assert "[full]" in line
|
||||
|
||||
def test_success_has_no_suffix(self):
|
||||
ok = json.dumps({"success": True, "data": "hi"})
|
||||
line = get_cute_tool_message("web_search", {"query": "hi"}, 0.2, result=ok)
|
||||
assert "[" not in line.split("0.2s", 1)[1]
|
||||
|
||||
def test_no_result_has_no_suffix(self):
|
||||
# No result passed at all — display function should not invent a
|
||||
# failure suffix.
|
||||
line = get_cute_tool_message("terminal", {"command": "ls"}, 0.2)
|
||||
assert "[" not in line.split("0.2s", 1)[1]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,156 @@
|
||||
"""Tests for external skill directories (skills.external_dirs config)."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def external_skills_dir(tmp_path):
|
||||
"""Create a temp dir with a sample external skill."""
|
||||
ext_dir = tmp_path / "external-skills"
|
||||
skill_dir = ext_dir / "my-external-skill"
|
||||
skill_dir.mkdir(parents=True)
|
||||
(skill_dir / "SKILL.md").write_text(
|
||||
"---\nname: my-external-skill\ndescription: A skill from an external directory\n---\n\n# My External Skill\n\nDo external things.\n"
|
||||
)
|
||||
return ext_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hermes_home(tmp_path):
|
||||
"""Create a minimal HERMES_HOME with config."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
(home / "skills").mkdir()
|
||||
return home
|
||||
|
||||
|
||||
class TestGetExternalSkillsDirs:
|
||||
def test_empty_config(self, hermes_home):
|
||||
(hermes_home / "config.yaml").write_text("skills:\n external_dirs: []\n")
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert result == []
|
||||
|
||||
def test_nonexistent_dir_skipped(self, hermes_home):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
"skills:\n external_dirs:\n - /nonexistent/path\n"
|
||||
)
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert result == []
|
||||
|
||||
def test_valid_dir_returned(self, hermes_home, external_skills_dir):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
|
||||
)
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert len(result) == 1
|
||||
assert result[0] == external_skills_dir.resolve()
|
||||
|
||||
def test_duplicate_dirs_deduplicated(self, hermes_home, external_skills_dir):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {external_skills_dir}\n - {external_skills_dir}\n"
|
||||
)
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert len(result) == 1
|
||||
|
||||
def test_local_skills_dir_excluded(self, hermes_home):
|
||||
local_skills = hermes_home / "skills"
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {local_skills}\n"
|
||||
)
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert result == []
|
||||
|
||||
def test_no_config_file(self, hermes_home):
|
||||
# No config.yaml at all
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert result == []
|
||||
|
||||
def test_string_value_converted_to_list(self, hermes_home, external_skills_dir):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs: {external_skills_dir}\n"
|
||||
)
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_external_skills_dirs
|
||||
result = get_external_skills_dirs()
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
class TestGetAllSkillsDirs:
|
||||
def test_local_always_first(self, hermes_home, external_skills_dir):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
|
||||
)
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}):
|
||||
from agent.skill_utils import get_all_skills_dirs
|
||||
result = get_all_skills_dirs()
|
||||
assert result[0] == hermes_home / "skills"
|
||||
assert result[1] == external_skills_dir.resolve()
|
||||
|
||||
|
||||
class TestExternalSkillsInFindAll:
|
||||
def test_external_skills_found(self, hermes_home, external_skills_dir):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
|
||||
)
|
||||
local_skills = hermes_home / "skills"
|
||||
with (
|
||||
patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}),
|
||||
patch("tools.skills_tool.SKILLS_DIR", local_skills),
|
||||
):
|
||||
from tools.skills_tool import _find_all_skills
|
||||
skills = _find_all_skills()
|
||||
names = [s["name"] for s in skills]
|
||||
assert "my-external-skill" in names
|
||||
|
||||
def test_local_takes_precedence(self, hermes_home, external_skills_dir):
|
||||
"""If the same skill name exists locally and externally, local wins."""
|
||||
local_skills = hermes_home / "skills"
|
||||
local_skill = local_skills / "my-external-skill"
|
||||
local_skill.mkdir(parents=True)
|
||||
(local_skill / "SKILL.md").write_text(
|
||||
"---\nname: my-external-skill\ndescription: Local version\n---\n\nLocal.\n"
|
||||
)
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
|
||||
)
|
||||
with (
|
||||
patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}),
|
||||
patch("tools.skills_tool.SKILLS_DIR", local_skills),
|
||||
):
|
||||
from tools.skills_tool import _find_all_skills
|
||||
skills = _find_all_skills()
|
||||
matching = [s for s in skills if s["name"] == "my-external-skill"]
|
||||
assert len(matching) == 1
|
||||
assert matching[0]["description"] == "Local version"
|
||||
|
||||
|
||||
class TestExternalSkillView:
|
||||
def test_skill_view_finds_external(self, hermes_home, external_skills_dir):
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {external_skills_dir}\n"
|
||||
)
|
||||
local_skills = hermes_home / "skills"
|
||||
with (
|
||||
patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}),
|
||||
patch("tools.skills_tool.SKILLS_DIR", local_skills),
|
||||
):
|
||||
from tools.skills_tool import skill_view
|
||||
result = json.loads(skill_view("my-external-skill"))
|
||||
assert result["success"] is True
|
||||
assert "external things" in result["content"]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Guards for ``get_external_skills_dirs`` mtime-based memo.
|
||||
|
||||
``get_external_skills_dirs()`` is called once per skill during banner
|
||||
construction and tool registration — on a typical install that's 120+
|
||||
calls. Without caching, each call re-reads + YAML-parses the full
|
||||
config.yaml (~85ms each, 10+ seconds total). This test pins the
|
||||
behavior: first call parses, subsequent calls return cached result,
|
||||
cache invalidates when config.yaml's mtime changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import skill_utils
|
||||
from agent.skill_utils import (
|
||||
_external_dirs_cache_clear,
|
||||
get_external_skills_dirs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hermes_home_with_config(tmp_path, monkeypatch):
|
||||
"""Isolated ``~/.hermes/`` with a config.yaml referencing one external dir."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
external = tmp_path / "external_skills"
|
||||
external.mkdir()
|
||||
|
||||
config = home / "config.yaml"
|
||||
config.write_text(
|
||||
"skills:\n"
|
||||
f" external_dirs:\n"
|
||||
f" - {external}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
_external_dirs_cache_clear()
|
||||
yield home, external, config
|
||||
_external_dirs_cache_clear()
|
||||
|
||||
|
||||
def test_returns_configured_external_dir(hermes_home_with_config):
|
||||
_home, external, _cfg = hermes_home_with_config
|
||||
result = get_external_skills_dirs()
|
||||
assert result == [external.resolve()]
|
||||
|
||||
|
||||
def test_cache_reuses_result_without_reparsing(hermes_home_with_config):
|
||||
"""Subsequent calls hit the cache and skip YAML parsing entirely."""
|
||||
_home, _external, _cfg = hermes_home_with_config
|
||||
|
||||
# Prime cache
|
||||
get_external_skills_dirs()
|
||||
|
||||
# Patch yaml_load to raise — if cache works, it's never called again.
|
||||
with patch.object(
|
||||
skill_utils,
|
||||
"yaml_load",
|
||||
side_effect=AssertionError("yaml_load should not run on cache hit"),
|
||||
):
|
||||
# Many calls, none should trigger the patched yaml_load.
|
||||
for _ in range(100):
|
||||
get_external_skills_dirs()
|
||||
|
||||
|
||||
def test_cache_invalidates_on_mtime_change(hermes_home_with_config):
|
||||
"""A config.yaml edit invalidates the cache on the next call."""
|
||||
_home, external, config = hermes_home_with_config
|
||||
other = external.parent / "other_skills"
|
||||
other.mkdir()
|
||||
|
||||
# Prime cache with original contents.
|
||||
first = get_external_skills_dirs()
|
||||
assert first == [external.resolve()]
|
||||
|
||||
# Rewrite config; bump mtime forward explicitly so filesystems with
|
||||
# coarse mtime granularity still register the change on fast test
|
||||
# systems.
|
||||
config.write_text(
|
||||
"skills:\n"
|
||||
f" external_dirs:\n"
|
||||
f" - {other}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
stat = config.stat()
|
||||
future = stat.st_atime + 10
|
||||
os.utime(config, (future, future))
|
||||
|
||||
second = get_external_skills_dirs()
|
||||
assert second == [other.resolve()]
|
||||
|
||||
|
||||
def test_returns_empty_when_config_missing(tmp_path, monkeypatch):
|
||||
"""No config file → empty list, cached as empty."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
_external_dirs_cache_clear()
|
||||
|
||||
assert get_external_skills_dirs() == []
|
||||
|
||||
|
||||
def test_returned_list_is_a_copy(hermes_home_with_config):
|
||||
"""Callers can't poison the cache by mutating the returned list."""
|
||||
first = get_external_skills_dirs()
|
||||
first.append(Path("/tmp/should-not-persist"))
|
||||
|
||||
second = get_external_skills_dirs()
|
||||
assert Path("/tmp/should-not-persist") not in second
|
||||
|
||||
|
||||
def test_cache_key_is_per_config_path(tmp_path, monkeypatch):
|
||||
"""Two different HERMES_HOMEs keep separate cache entries."""
|
||||
home_a = tmp_path / "home_a" / ".hermes"
|
||||
home_a.mkdir(parents=True)
|
||||
ext_a = tmp_path / "ext_a"
|
||||
ext_a.mkdir()
|
||||
(home_a / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {ext_a}\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
home_b = tmp_path / "home_b" / ".hermes"
|
||||
home_b.mkdir(parents=True)
|
||||
ext_b = tmp_path / "ext_b"
|
||||
ext_b.mkdir()
|
||||
(home_b / "config.yaml").write_text(
|
||||
f"skills:\n external_dirs:\n - {ext_b}\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
_external_dirs_cache_clear()
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(home_a))
|
||||
assert get_external_skills_dirs() == [ext_a.resolve()]
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(home_b))
|
||||
assert get_external_skills_dirs() == [ext_b.resolve()]
|
||||
|
||||
# And switching back still works — both entries coexist in the cache.
|
||||
monkeypatch.setenv("HERMES_HOME", str(home_a))
|
||||
assert get_external_skills_dirs() == [ext_a.resolve()]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Tests for agent/file_safety.py read guards — env file blocking.
|
||||
|
||||
Run with: python -m pytest tests/agent/test_file_safety.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.file_safety import (
|
||||
_BLOCKED_PROJECT_ENV_BASENAMES,
|
||||
get_read_block_error,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project-local .env file blocking (issue #20734)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvFileReadBlocking:
|
||||
"""Secret-bearing .env files must be blocked by get_read_block_error."""
|
||||
|
||||
@pytest.mark.parametrize("basename", [
|
||||
".env",
|
||||
".env.local",
|
||||
".env.development",
|
||||
".env.production",
|
||||
".env.test",
|
||||
".env.staging",
|
||||
".envrc",
|
||||
])
|
||||
def test_blocked_env_basenames(self, basename):
|
||||
"""All secret-bearing .env basenames are blocked regardless of directory."""
|
||||
path = f"/tmp/project/{basename}"
|
||||
error = get_read_block_error(path)
|
||||
assert error is not None, f"{basename} should be blocked"
|
||||
assert "Access denied" in error
|
||||
assert "secret-bearing" in error.lower() or "environment file" in error.lower()
|
||||
|
||||
def test_blocked_env_in_subdirectory(self):
|
||||
"""Nested .env files are also blocked."""
|
||||
error = get_read_block_error("/home/user/app/services/api/.env.production")
|
||||
assert error is not None
|
||||
|
||||
def test_blocked_env_absolute_path(self):
|
||||
"""Absolute paths to .env files are blocked."""
|
||||
error = get_read_block_error("/opt/myapp/.env")
|
||||
assert error is not None
|
||||
|
||||
def test_allowed_env_example(self):
|
||||
""""The .env.example file is explicitly allowed — it's documentation, not a secret."""
|
||||
error = get_read_block_error("/tmp/project/.env.example")
|
||||
assert error is None
|
||||
|
||||
def test_allowed_env_sample(self):
|
||||
"""Other .env variants like .env.sample are allowed."""
|
||||
error = get_read_block_error("/tmp/project/.env.sample")
|
||||
assert error is None
|
||||
|
||||
def test_allowed_non_env_files(self):
|
||||
"""Regular files are not affected by the env guard."""
|
||||
for path in ["/tmp/project/config.yaml", "/tmp/project/main.py",
|
||||
"/tmp/project/README.md", "/tmp/project/.gitignore"]:
|
||||
error = get_read_block_error(path)
|
||||
assert error is None, f"{path} should be allowed"
|
||||
|
||||
def test_allowed_hermes_env(self):
|
||||
"""Hermes' own .env inside HERMES_HOME is NOT blocked by this rule
|
||||
(it's handled by other mechanisms). Only project-local .env is blocked."""
|
||||
# Note: hermes internal .env is in ~/.hermes/.env which is NOT a project-local
|
||||
# path, but the basename check applies to ANY .env. This is intentional —
|
||||
# even ~/.hermes/.env should not be readable via read_file.
|
||||
error = get_read_block_error(os.path.expanduser("~/.hermes/.env"))
|
||||
assert error is not None
|
||||
|
||||
def test_blocked_set_is_lowercase(self):
|
||||
"""All entries in the blocked set are lowercase for case-insensitive matching."""
|
||||
for name in _BLOCKED_PROJECT_ENV_BASENAMES:
|
||||
assert name == name.lower(), f"{name} should be lowercase"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Existing cache-file blocking (regression — must still work)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCacheFileReadBlocking:
|
||||
"""Internal Hermes cache files must remain blocked."""
|
||||
|
||||
def test_hub_index_cache_blocked(self, tmp_path):
|
||||
"""Hub index-cache reads are blocked."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
cache = hermes_home / "skills" / ".hub" / "index-cache" / "data.json"
|
||||
cache.parent.mkdir(parents=True)
|
||||
cache.write_text("{}")
|
||||
|
||||
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
|
||||
error = get_read_block_error(str(cache))
|
||||
assert error is not None
|
||||
assert "internal Hermes cache" in error
|
||||
|
||||
def test_hub_directory_blocked(self, tmp_path):
|
||||
"""Hub directory reads are blocked."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hub = hermes_home / "skills" / ".hub" / "metadata.json"
|
||||
hub.parent.mkdir(parents=True)
|
||||
hub.write_text("{}")
|
||||
|
||||
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
|
||||
error = get_read_block_error(str(hub))
|
||||
assert error is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Combined: env guard + cache guard don't interfere
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCombinedGuards:
|
||||
"""Both guards should work independently without interference."""
|
||||
|
||||
def test_env_guard_works_regardless_of_hermes_home(self, tmp_path):
|
||||
"""The env basename guard does not depend on HERMES_HOME resolution."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
|
||||
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
|
||||
# Regular project .env should still be blocked
|
||||
error = get_read_block_error("/workspace/.env")
|
||||
assert error is not None
|
||||
|
||||
# .env.example should still be allowed
|
||||
error = get_read_block_error("/workspace/.env.example")
|
||||
assert error is None
|
||||
|
||||
def test_cache_guard_still_works_with_env_guard(self, tmp_path):
|
||||
"""Cache file blocking still works when env guard is active."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
cache = hermes_home / "skills" / ".hub" / "index-cache" / "x"
|
||||
cache.parent.mkdir(parents=True)
|
||||
cache.write_text("")
|
||||
|
||||
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
|
||||
error = get_read_block_error(str(cache))
|
||||
assert error is not None
|
||||
assert "internal Hermes cache" in error
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Tests for HERMES_HOME credential-file read blocking in file_safety.
|
||||
|
||||
Regression for https://github.com/NousResearch/hermes-agent/issues/17656 —
|
||||
``read_file`` was previously only sandboxed against ``HERMES_HOME`` itself,
|
||||
which left ``auth.json`` and ``.anthropic_oauth.json`` (plaintext provider
|
||||
keys + OAuth tokens) readable by the agent. A prompt-injection reaching
|
||||
``read_file`` could exfiltrate active credentials.
|
||||
|
||||
These tests verify that ``get_read_block_error`` returns a denial message
|
||||
for the credential stores while leaving arbitrary ``HERMES_HOME`` files
|
||||
readable, and that the existing ``skills/.hub`` deny still applies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fake_home(tmp_path, monkeypatch):
|
||||
"""Point ``_hermes_home_path()`` at a tmp dir for isolated checks."""
|
||||
import agent.file_safety as fs
|
||||
|
||||
home = tmp_path / "hermes_home"
|
||||
home.mkdir()
|
||||
monkeypatch.setattr(fs, "_hermes_home_path", lambda: home)
|
||||
return home
|
||||
|
||||
|
||||
def _create(home: Path, rel: str | Path) -> Path:
|
||||
"""Create the file (with parents) so realpath() resolves it."""
|
||||
p = home / rel
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text("dummy", encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def test_auth_json_blocked(fake_home):
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
auth = _create(fake_home, "auth.json")
|
||||
err = get_read_block_error(str(auth))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
assert "auth.json" in err
|
||||
|
||||
|
||||
def test_auth_lock_blocked(fake_home):
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
lock = _create(fake_home, "auth.lock")
|
||||
err = get_read_block_error(str(lock))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_anthropic_oauth_json_blocked(fake_home):
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
oauth = _create(fake_home, ".anthropic_oauth.json")
|
||||
err = get_read_block_error(str(oauth))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_google_oauth_json_blocked(fake_home):
|
||||
"""Gemini OAuth tokens live under auth/google_oauth.json — blocked."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
oauth = _create(fake_home, Path("auth") / "google_oauth.json")
|
||||
err = get_read_block_error(str(oauth))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_arbitrary_hermes_home_file_not_blocked(fake_home):
|
||||
"""Non-credential files inside HERMES_HOME stay readable."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
safe = _create(fake_home, "session_log.txt")
|
||||
assert get_read_block_error(str(safe)) is None
|
||||
|
||||
|
||||
def test_subdirectory_named_auth_json_not_blocked(fake_home):
|
||||
"""Only the top-level auth.json is the credential store; a file with the
|
||||
same name in a subdirectory (e.g., a skill mock) must remain readable."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
nested = _create(fake_home, Path("skills") / "my-skill" / "auth.json")
|
||||
assert get_read_block_error(str(nested)) is None
|
||||
|
||||
|
||||
def test_skills_hub_block_still_applies(fake_home):
|
||||
"""Regression guard: the original skills/.hub deny must keep working."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
hub_file = _create(fake_home, "skills/.hub/manifest.json")
|
||||
err = get_read_block_error(str(hub_file))
|
||||
assert err is not None
|
||||
assert "internal Hermes cache file" in err
|
||||
|
||||
|
||||
def test_path_traversal_resolves_to_blocked(fake_home, tmp_path):
|
||||
"""A path that traverses through a sibling dir back into HERMES_HOME's
|
||||
auth.json must still be caught — the check resolves through realpath."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
_create(fake_home, "auth.json")
|
||||
sibling = tmp_path / "elsewhere"
|
||||
sibling.mkdir()
|
||||
traversal = sibling / ".." / "hermes_home" / "auth.json"
|
||||
err = get_read_block_error(str(traversal))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_symlink_to_auth_json_blocked(fake_home, tmp_path):
|
||||
"""A symlink pointing at HERMES_HOME/auth.json from outside the home
|
||||
must be blocked — readlink-resolution catches the indirection."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
target = _create(fake_home, "auth.json")
|
||||
link = tmp_path / "shim.json"
|
||||
try:
|
||||
os.symlink(target, link)
|
||||
except (OSError, NotImplementedError):
|
||||
pytest.skip("symlinks not supported on this platform/filesystem")
|
||||
err = get_read_block_error(str(link))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_read_file_tool_blocks_relative_path_under_terminal_cwd(
|
||||
fake_home, tmp_path, monkeypatch
|
||||
):
|
||||
"""Bypass guard: a relative path like ``"auth.json"`` resolved by
|
||||
``read_file_tool`` against ``TERMINAL_CWD == HERMES_HOME`` must still
|
||||
be blocked, even though ``get_read_block_error``'s own ``resolve()``
|
||||
is anchored at the (different) Python process cwd.
|
||||
"""
|
||||
import json
|
||||
|
||||
import tools.file_tools as ft
|
||||
|
||||
_create(fake_home, "auth.json")
|
||||
# Force the file_tools resolver to anchor relative paths at HERMES_HOME
|
||||
# while the Python process cwd remains tmp_path (a different directory).
|
||||
monkeypatch.setenv("TERMINAL_CWD", str(fake_home))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(
|
||||
ft, "_get_live_tracking_cwd", lambda task_id="default": None
|
||||
)
|
||||
|
||||
out = json.loads(ft.read_file_tool("auth.json"))
|
||||
assert "error" in out
|
||||
assert "credential store" in out["error"]
|
||||
|
||||
|
||||
def test_read_file_tool_blocks_nested_google_oauth_path(
|
||||
fake_home, tmp_path, monkeypatch
|
||||
):
|
||||
"""The real read_file tool must not return Gemini OAuth token material."""
|
||||
import json
|
||||
|
||||
import tools.file_tools as ft
|
||||
|
||||
oauth = _create(fake_home, Path("auth") / "google_oauth.json")
|
||||
oauth.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"refresh": "REFRESH_TOKEN_MARKER",
|
||||
"access": "ACCESS_TOKEN_MARKER",
|
||||
"email": "user@example.com",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(
|
||||
ft, "_get_live_tracking_cwd", lambda task_id="default": None
|
||||
)
|
||||
|
||||
out = json.loads(ft.read_file_tool(str(oauth), task_id="google-oauth-test"))
|
||||
assert "error" in out
|
||||
assert "credential store" in out["error"]
|
||||
assert "REFRESH_TOKEN_MARKER" not in json.dumps(out)
|
||||
assert "ACCESS_TOKEN_MARKER" not in json.dumps(out)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Widening: .env, webhook_subscriptions.json, mcp-tokens/
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_dotenv_blocked(fake_home):
|
||||
""".env in HERMES_HOME holds API keys — blocked."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
env = _create(fake_home, ".env")
|
||||
err = get_read_block_error(str(env))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_webhook_subscriptions_blocked(fake_home):
|
||||
"""webhook_subscriptions.json holds per-route HMAC secrets — blocked."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
subs = _create(fake_home, "webhook_subscriptions.json")
|
||||
err = get_read_block_error(str(subs))
|
||||
assert err is not None
|
||||
assert "credential store" in err
|
||||
|
||||
|
||||
def test_mcp_tokens_file_blocked(fake_home):
|
||||
"""Files under mcp-tokens/ hold OAuth tokens — blocked."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
tok = _create(fake_home, Path("mcp-tokens") / "github.json")
|
||||
err = get_read_block_error(str(tok))
|
||||
assert err is not None
|
||||
assert "MCP token" in err
|
||||
|
||||
|
||||
def test_mcp_tokens_nested_blocked(fake_home):
|
||||
"""Nested files inside mcp-tokens/ are also blocked."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
tok = _create(fake_home, Path("mcp-tokens") / "providers" / "azure.json")
|
||||
err = get_read_block_error(str(tok))
|
||||
assert err is not None
|
||||
assert "MCP token" in err
|
||||
|
||||
|
||||
def test_mcp_tokens_dir_itself_blocked(fake_home):
|
||||
"""The mcp-tokens directory itself is blocked (listing is exfiltrating)."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
tokens_dir = fake_home / "mcp-tokens"
|
||||
tokens_dir.mkdir(parents=True, exist_ok=True)
|
||||
err = get_read_block_error(str(tokens_dir))
|
||||
assert err is not None
|
||||
assert "MCP token" in err
|
||||
|
||||
|
||||
def test_identically_named_hermes_files_outside_home_not_blocked(
|
||||
fake_home, tmp_path
|
||||
):
|
||||
"""Hermes-specific filenames (``auth.json``, ``mcp-tokens/``, ``google_oauth.json``)
|
||||
outside HERMES_HOME must remain readable — the gate is per-location for
|
||||
those, not per-filename. ``.env`` is the exception: it's blocked anywhere
|
||||
on disk (see test_project_local_env_blocked) because the basename always
|
||||
means \"secret-bearing environment file\" regardless of directory."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
project = tmp_path / "myproject"
|
||||
project.mkdir()
|
||||
# auth.json outside HERMES_HOME — readable (per-location gate).
|
||||
p = project / "auth.json"
|
||||
p.write_text("not secret here", encoding="utf-8")
|
||||
assert get_read_block_error(str(p)) is None, (
|
||||
"auth.json outside HERMES_HOME should NOT be blocked"
|
||||
)
|
||||
|
||||
google_oauth = project / "auth" / "google_oauth.json"
|
||||
google_oauth.parent.mkdir()
|
||||
google_oauth.write_text("not really a token", encoding="utf-8")
|
||||
assert get_read_block_error(str(google_oauth)) is None
|
||||
|
||||
tokens = project / "mcp-tokens"
|
||||
tokens.mkdir()
|
||||
tok_file = tokens / "token.json"
|
||||
tok_file.write_text("not really a token", encoding="utf-8")
|
||||
assert get_read_block_error(str(tok_file)) is None
|
||||
|
||||
|
||||
def test_non_secret_auth_subtree_file_not_blocked(fake_home):
|
||||
"""Only the known Google OAuth token path is blocked, not all auth/*."""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
note = _create(fake_home, Path("auth") / "notes.json")
|
||||
assert get_read_block_error(str(note)) is None
|
||||
|
||||
|
||||
def test_config_yaml_not_blocked(fake_home):
|
||||
"""config.yaml is NOT a credential file — agent should still be
|
||||
able to read it for debugging. (Writes are denied separately by
|
||||
is_write_denied; reads stay allowed.)"""
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
cfg = _create(fake_home, "config.yaml")
|
||||
assert get_read_block_error(str(cfg)) is None
|
||||
|
||||
|
||||
def test_profile_mode_blocks_root_credentials(tmp_path, monkeypatch):
|
||||
"""Under a profile, HERMES_HOME = <root>/profiles/<name>, but
|
||||
<root>/auth.json must ALSO be blocked — credentials at root are
|
||||
inherited by every profile."""
|
||||
import agent.file_safety as fs
|
||||
|
||||
root = tmp_path / "hermes"
|
||||
profile = root / "profiles" / "coder"
|
||||
profile.mkdir(parents=True)
|
||||
monkeypatch.setattr(fs, "_hermes_home_path", lambda: profile)
|
||||
monkeypatch.setattr(fs, "_hermes_root_path", lambda: root)
|
||||
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
# Profile-local credential store: blocked
|
||||
profile_auth = profile / "auth.json"
|
||||
profile_auth.write_text("x")
|
||||
assert "credential store" in (get_read_block_error(str(profile_auth)) or "")
|
||||
|
||||
# Root-level credential store: ALSO blocked (this is the widening)
|
||||
root_auth = root / "auth.json"
|
||||
root_auth.write_text("x")
|
||||
assert "credential store" in (get_read_block_error(str(root_auth)) or "")
|
||||
|
||||
# Root-level .env: blocked too
|
||||
root_env = root / ".env"
|
||||
root_env.write_text("x")
|
||||
assert "credential store" in (get_read_block_error(str(root_env)) or "")
|
||||
|
||||
# Root-level Google OAuth token store: blocked too
|
||||
root_google_oauth = root / "auth" / "google_oauth.json"
|
||||
root_google_oauth.parent.mkdir(parents=True, exist_ok=True)
|
||||
root_google_oauth.write_text("x")
|
||||
assert "credential store" in (
|
||||
get_read_block_error(str(root_google_oauth)) or ""
|
||||
)
|
||||
|
||||
# Root-level mcp-tokens: blocked
|
||||
root_tok = root / "mcp-tokens" / "gh.json"
|
||||
root_tok.parent.mkdir(parents=True, exist_ok=True)
|
||||
root_tok.write_text("x")
|
||||
assert "MCP token" in (get_read_block_error(str(root_tok)) or "")
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Tests for the cross-Hermes-profile write guard in agent/file_safety.
|
||||
|
||||
The guard fires when a tool tries to write into another Hermes profile's
|
||||
skills/plugins/cron/memories directory. It's a soft guard — defense in
|
||||
depth, NOT a security boundary — but it prevents the agent from silently
|
||||
corrupting a profile that belongs to a different session.
|
||||
|
||||
Reference: May 2026 incident — a hermes-security profile session
|
||||
accidentally edited skills under both ~/.hermes/profiles/hermes-security/skills/
|
||||
AND ~/.hermes/skills/ (the default profile's skills), realizing only
|
||||
afterwards that the second path belonged to a different profile.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers — set up a fake Hermes root with two profiles, monkeypatch the
|
||||
# resolver helpers so the classifier sees the test layout.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_hermes(tmp_path, monkeypatch):
|
||||
"""Build a fake Hermes layout:
|
||||
|
||||
<tmp>/
|
||||
skills/foo/SKILL.md # default profile
|
||||
plugins/foo/__init__.py
|
||||
cron/<state>
|
||||
memories/MEMORY.md
|
||||
profiles/
|
||||
hermes-security/
|
||||
skills/foo/SKILL.md # named profile
|
||||
plugins/...
|
||||
coder/
|
||||
skills/foo/SKILL.md # another named profile
|
||||
"""
|
||||
root = tmp_path / "fake-hermes"
|
||||
(root / "skills" / "foo").mkdir(parents=True)
|
||||
(root / "skills" / "foo" / "SKILL.md").write_text("# default skill\n")
|
||||
(root / "plugins" / "foo").mkdir(parents=True)
|
||||
(root / "memories").mkdir(parents=True)
|
||||
(root / "cron").mkdir(parents=True)
|
||||
|
||||
sec_home = root / "profiles" / "hermes-security"
|
||||
(sec_home / "skills" / "foo").mkdir(parents=True)
|
||||
(sec_home / "skills" / "foo" / "SKILL.md").write_text("# sec skill\n")
|
||||
(sec_home / "plugins").mkdir(parents=True)
|
||||
|
||||
coder_home = root / "profiles" / "coder"
|
||||
(coder_home / "skills" / "foo").mkdir(parents=True)
|
||||
(coder_home / "skills" / "foo" / "SKILL.md").write_text("# coder skill\n")
|
||||
|
||||
# Monkeypatch the resolver functions used by file_safety so each test
|
||||
# can choose which profile is "active".
|
||||
import hermes_constants
|
||||
monkeypatch.setattr(hermes_constants, "get_default_hermes_root", lambda: root)
|
||||
|
||||
# The reloads below ensure get_cross_profile_warning/classify see the patched root.
|
||||
import agent.file_safety as fs
|
||||
monkeypatch.setattr(fs, "_hermes_root_path", lambda: root)
|
||||
|
||||
return {
|
||||
"root": root,
|
||||
"default_home": root,
|
||||
"security_home": sec_home,
|
||||
"coder_home": coder_home,
|
||||
}
|
||||
|
||||
|
||||
def _set_active_home(monkeypatch, hermes_home: Path):
|
||||
"""Point file_safety._hermes_home_path at a specific profile dir."""
|
||||
import agent.file_safety as fs
|
||||
monkeypatch.setattr(fs, "_hermes_home_path", lambda: hermes_home)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _resolve_active_profile_name
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveActiveProfileName:
|
||||
def test_default_when_home_is_root(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["default_home"])
|
||||
from agent.file_safety import _resolve_active_profile_name
|
||||
assert _resolve_active_profile_name() == "default"
|
||||
|
||||
def test_named_profile(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import _resolve_active_profile_name
|
||||
assert _resolve_active_profile_name() == "hermes-security"
|
||||
|
||||
def test_falls_back_to_default_on_resolution_failure(self, fake_hermes, monkeypatch):
|
||||
"""If HERMES_HOME resolution raises, return 'default' rather than crashing the tool."""
|
||||
import agent.file_safety as fs
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("simulated")
|
||||
|
||||
monkeypatch.setattr(fs, "_hermes_home_path", _boom)
|
||||
# Should not raise — falls back to "default"
|
||||
assert fs._resolve_active_profile_name() == "default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# classify_cross_profile_target
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestClassifyCrossProfileTarget:
|
||||
def test_same_profile_write_returns_none(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
result = classify_cross_profile_target(
|
||||
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_security_writing_default_skill(self, fake_hermes, monkeypatch):
|
||||
"""The exact incident from May 2026."""
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
result = classify_cross_profile_target(
|
||||
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
|
||||
)
|
||||
assert result is not None
|
||||
assert result["active_profile"] == "hermes-security"
|
||||
assert result["target_profile"] == "default"
|
||||
assert result["area"] == "skills"
|
||||
|
||||
def test_default_writing_security_skill(self, fake_hermes, monkeypatch):
|
||||
"""Inverse direction — default-profile session reaching into a named profile."""
|
||||
_set_active_home(monkeypatch, fake_hermes["default_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
result = classify_cross_profile_target(
|
||||
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
|
||||
)
|
||||
assert result is not None
|
||||
assert result["active_profile"] == "default"
|
||||
assert result["target_profile"] == "hermes-security"
|
||||
|
||||
def test_named_to_named_cross_profile(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
result = classify_cross_profile_target(
|
||||
str(fake_hermes["coder_home"] / "skills" / "foo" / "SKILL.md")
|
||||
)
|
||||
assert result is not None
|
||||
assert result["target_profile"] == "coder"
|
||||
|
||||
@pytest.mark.parametrize("area", ["skills", "plugins", "cron", "memories"])
|
||||
def test_all_profile_scoped_areas_classified(self, fake_hermes, monkeypatch, area):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
target = fake_hermes["default_home"] / area / "foo.txt"
|
||||
result = classify_cross_profile_target(str(target))
|
||||
assert result is not None
|
||||
assert result["area"] == area
|
||||
|
||||
def test_non_hermes_path_returns_none(self, fake_hermes, monkeypatch, tmp_path):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
# Path outside any Hermes root
|
||||
assert classify_cross_profile_target(str(tmp_path / "random.txt")) is None
|
||||
|
||||
def test_hermes_config_not_classified_as_cross_profile(self, fake_hermes, monkeypatch):
|
||||
"""Files under <root>/config.yaml or <root>/.env are NOT profile-scoped
|
||||
(already covered by build_write_denied_paths). Don't double-warn."""
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import classify_cross_profile_target
|
||||
# config.yaml at root level is not in PROFILE_SCOPED_AREAS
|
||||
result = classify_cross_profile_target(
|
||||
str(fake_hermes["default_home"] / "config.yaml")
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_cross_profile_warning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetCrossProfileWarning:
|
||||
def test_in_profile_returns_none(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import get_cross_profile_warning
|
||||
assert get_cross_profile_warning(
|
||||
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
|
||||
) is None
|
||||
|
||||
def test_cross_profile_warning_names_both_profiles(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import get_cross_profile_warning
|
||||
warn = get_cross_profile_warning(
|
||||
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
|
||||
)
|
||||
assert warn is not None
|
||||
# Must name BOTH profiles so the model knows which is which.
|
||||
assert "default" in warn
|
||||
assert "hermes-security" in warn
|
||||
# Must name the bypass kwarg.
|
||||
assert "cross_profile=True" in warn
|
||||
# Must reference the area.
|
||||
assert "skills" in warn
|
||||
|
||||
def test_warning_is_defense_in_depth_not_boundary(self, fake_hermes, monkeypatch):
|
||||
_set_active_home(monkeypatch, fake_hermes["security_home"])
|
||||
from agent.file_safety import get_cross_profile_warning
|
||||
warn = get_cross_profile_warning(
|
||||
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
|
||||
)
|
||||
# Must self-document as defense-in-depth so future reviewers
|
||||
# don't promote it to a hard block.
|
||||
assert "not a security boundary" in warn.lower()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
"""Regression tests for #13636 — CloudCode / Gemini CLI rate-limit fallback.
|
||||
|
||||
_pool_may_recover_from_rate_limit() is the hinge between credential-pool
|
||||
rotation and fallback-provider activation. For CloudCode (Gemini CLI /
|
||||
Gemini OAuth) the 429 is an account-wide throttle, so waiting for pool
|
||||
rotation is pointless — prefer fallback immediately.
|
||||
"""
|
||||
import inspect
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from agent import conversation_loop
|
||||
from run_agent import _pool_may_recover_from_rate_limit
|
||||
|
||||
|
||||
def _pool(entries: int = 2):
|
||||
p = MagicMock()
|
||||
p.has_available.return_value = True
|
||||
p.entries.return_value = list(range(entries))
|
||||
return p
|
||||
|
||||
|
||||
def test_cloudcode_provider_skips_pool_rotation():
|
||||
assert _pool_may_recover_from_rate_limit(
|
||||
_pool(entries=3),
|
||||
provider="google-gemini-cli",
|
||||
base_url="cloudcode-pa://google",
|
||||
) is False
|
||||
|
||||
|
||||
def test_cloudcode_base_url_skips_pool_rotation_even_on_alias_provider():
|
||||
# Even if the provider label is something else, a cloudcode-pa:// URL
|
||||
# signals the account-wide quota regime.
|
||||
assert _pool_may_recover_from_rate_limit(
|
||||
_pool(entries=3),
|
||||
provider="custom-provider",
|
||||
base_url="cloudcode-pa://google",
|
||||
) is False
|
||||
|
||||
|
||||
def test_non_cloudcode_multi_entry_pool_still_recovers():
|
||||
assert _pool_may_recover_from_rate_limit(
|
||||
_pool(entries=3),
|
||||
provider="openrouter",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
) is True
|
||||
|
||||
|
||||
def test_single_entry_pool_skips_rotation_regardless_of_provider():
|
||||
# Pre-existing single-entry-pool exception (#11314) still holds.
|
||||
assert _pool_may_recover_from_rate_limit(
|
||||
_pool(entries=1),
|
||||
provider="openrouter",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
) is False
|
||||
|
||||
|
||||
def test_exhausted_pool_skips_rotation():
|
||||
p = MagicMock()
|
||||
p.has_available.return_value = False
|
||||
assert _pool_may_recover_from_rate_limit(p) is False
|
||||
|
||||
|
||||
def test_no_pool_skips_rotation():
|
||||
assert _pool_may_recover_from_rate_limit(None) is False
|
||||
|
||||
|
||||
def test_conversation_loop_resolves_pool_helper_through_run_agent_module():
|
||||
"""Extracted conversation loop must honor tests/patches on run_agent.
|
||||
|
||||
conversation_loop intentionally lazy-loads run_agent via _ra(). If this
|
||||
call site uses a bare imported helper, monkeypatching run_agent in tests (and
|
||||
production wrappers that patch run_agent) will not propagate into the
|
||||
extracted loop; older code also hit NameError in this branch.
|
||||
"""
|
||||
source = inspect.getsource(conversation_loop.run_conversation)
|
||||
|
||||
assert "_ra()._pool_may_recover_from_rate_limit(" in source
|
||||
assert "pool_may_recover = _pool_may_recover_from_rate_limit(" not in source
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Tests for Gemini free-tier detection and blocking."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
from agent.gemini_native_adapter import (
|
||||
gemini_http_error,
|
||||
is_free_tier_quota_error,
|
||||
probe_gemini_tier,
|
||||
)
|
||||
|
||||
|
||||
def _mock_response(status: int, headers: dict | None = None, text: str = "") -> MagicMock:
|
||||
resp = MagicMock()
|
||||
resp.status_code = status
|
||||
resp.headers = headers or {}
|
||||
resp.text = text
|
||||
return resp
|
||||
|
||||
|
||||
def _run_probe(resp: MagicMock) -> str:
|
||||
with patch("agent.gemini_native_adapter.httpx.Client") as MC:
|
||||
inst = MagicMock()
|
||||
inst.post.return_value = resp
|
||||
MC.return_value.__enter__.return_value = inst
|
||||
return probe_gemini_tier("fake-key")
|
||||
|
||||
|
||||
class TestProbeGeminiTier:
|
||||
"""Verify the tier probe classifies keys correctly."""
|
||||
|
||||
def test_free_tier_via_rpd_header_flash(self):
|
||||
# gemini-2.5-flash free tier: 250 RPD
|
||||
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "250"}, "{}")
|
||||
assert _run_probe(resp) == "free"
|
||||
|
||||
def test_free_tier_via_rpd_header_pro(self):
|
||||
# gemini-2.5-pro free tier: 100 RPD
|
||||
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "100"}, "{}")
|
||||
assert _run_probe(resp) == "free"
|
||||
|
||||
def test_free_tier_via_rpd_header_flash_lite(self):
|
||||
# flash-lite free tier: 1000 RPD (our upper bound)
|
||||
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "1000"}, "{}")
|
||||
assert _run_probe(resp) == "free"
|
||||
|
||||
def test_paid_tier_via_rpd_header(self):
|
||||
# Tier 1 starts at 1500+ RPD
|
||||
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "1500"}, "{}")
|
||||
assert _run_probe(resp) == "paid"
|
||||
|
||||
def test_free_tier_via_429_body(self):
|
||||
body = (
|
||||
'{"error":{"code":429,"message":"Quota exceeded for metric: '
|
||||
'generativelanguage.googleapis.com/generate_content_free_tier_requests, '
|
||||
'limit: 20"}}'
|
||||
)
|
||||
resp = _mock_response(429, {}, body)
|
||||
assert _run_probe(resp) == "free"
|
||||
|
||||
def test_paid_429_has_no_free_tier_marker(self):
|
||||
body = '{"error":{"code":429,"message":"rate limited"}}'
|
||||
resp = _mock_response(429, {}, body)
|
||||
assert _run_probe(resp) == "paid"
|
||||
|
||||
def test_successful_200_without_rpd_header_is_paid(self):
|
||||
resp = _mock_response(200, {}, '{"candidates":[]}')
|
||||
assert _run_probe(resp) == "paid"
|
||||
|
||||
def test_401_returns_unknown(self):
|
||||
resp = _mock_response(401, {}, '{"error":{"code":401}}')
|
||||
assert _run_probe(resp) == "unknown"
|
||||
|
||||
def test_404_returns_unknown(self):
|
||||
resp = _mock_response(404, {}, '{"error":{"code":404}}')
|
||||
assert _run_probe(resp) == "unknown"
|
||||
|
||||
def test_network_error_returns_unknown(self):
|
||||
with patch(
|
||||
"agent.gemini_native_adapter.httpx.Client",
|
||||
side_effect=Exception("dns failure"),
|
||||
):
|
||||
assert probe_gemini_tier("fake-key") == "unknown"
|
||||
|
||||
def test_empty_key_returns_unknown(self):
|
||||
assert probe_gemini_tier("") == "unknown"
|
||||
assert probe_gemini_tier(" ") == "unknown"
|
||||
assert probe_gemini_tier(None) == "unknown" # type: ignore[arg-type]
|
||||
|
||||
def test_malformed_rpd_header_falls_through(self):
|
||||
# Non-integer header value shouldn't crash; 200 with no usable header -> paid.
|
||||
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "abc"}, "{}")
|
||||
assert _run_probe(resp) == "paid"
|
||||
|
||||
def test_openai_compat_suffix_stripped(self):
|
||||
"""Base URLs ending in /openai get normalized to the native endpoint."""
|
||||
resp = _mock_response(200, {"x-ratelimit-limit-requests-per-day": "1500"}, "{}")
|
||||
with patch("agent.gemini_native_adapter.httpx.Client") as MC:
|
||||
inst = MagicMock()
|
||||
inst.post.return_value = resp
|
||||
MC.return_value.__enter__.return_value = inst
|
||||
probe_gemini_tier(
|
||||
"fake",
|
||||
"https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
)
|
||||
# Verify the post URL does NOT contain /openai
|
||||
called_url = inst.post.call_args[0][0]
|
||||
assert "/openai/" not in called_url
|
||||
assert called_url.endswith(":generateContent")
|
||||
|
||||
|
||||
class TestIsFreeTierQuotaError:
|
||||
def test_detects_free_tier_marker(self):
|
||||
assert is_free_tier_quota_error(
|
||||
"Quota exceeded for metric: generate_content_free_tier_requests"
|
||||
)
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert is_free_tier_quota_error("QUOTA: FREE_TIER_REQUESTS")
|
||||
|
||||
def test_no_free_tier_marker(self):
|
||||
assert not is_free_tier_quota_error("rate limited")
|
||||
|
||||
def test_empty_string(self):
|
||||
assert not is_free_tier_quota_error("")
|
||||
|
||||
def test_none(self):
|
||||
assert not is_free_tier_quota_error(None) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestGeminiHttpErrorFreeTierGuidance:
|
||||
"""gemini_http_error should append free-tier guidance for free-tier 429s."""
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, status: int, text: str):
|
||||
self.status_code = status
|
||||
self.headers: dict = {}
|
||||
self.text = text
|
||||
|
||||
def test_free_tier_429_appends_guidance(self):
|
||||
body = (
|
||||
'{"error":{"code":429,"message":"Quota exceeded for metric: '
|
||||
"generativelanguage.googleapis.com/generate_content_free_tier_requests, "
|
||||
'limit: 20","status":"RESOURCE_EXHAUSTED"}}'
|
||||
)
|
||||
err = gemini_http_error(self._FakeResp(429, body))
|
||||
msg = str(err)
|
||||
assert "free tier" in msg.lower()
|
||||
assert "aistudio.google.com/apikey" in msg
|
||||
|
||||
def test_paid_429_has_no_billing_url(self):
|
||||
body = '{"error":{"code":429,"message":"Rate limited","status":"RESOURCE_EXHAUSTED"}}'
|
||||
err = gemini_http_error(self._FakeResp(429, body))
|
||||
assert "aistudio.google.com/apikey" not in str(err)
|
||||
|
||||
def test_non_429_has_no_billing_url(self):
|
||||
body = '{"error":{"code":400,"message":"bad request","status":"INVALID_ARGUMENT"}}'
|
||||
err = gemini_http_error(self._FakeResp(400, body))
|
||||
assert "aistudio.google.com/apikey" not in str(err)
|
||||
|
||||
def test_401_has_no_billing_url(self):
|
||||
body = '{"error":{"code":401,"message":"API key invalid","status":"UNAUTHENTICATED"}}'
|
||||
err = gemini_http_error(self._FakeResp(401, body))
|
||||
assert "aistudio.google.com/apikey" not in str(err)
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Tests for the native Google AI Studio Gemini adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyResponse:
|
||||
def __init__(self, status_code=200, payload=None, headers=None, text=None):
|
||||
self.status_code = status_code
|
||||
self._payload = payload or {}
|
||||
self.headers = headers or {}
|
||||
self.text = text if text is not None else json.dumps(self._payload)
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def test_build_native_request_preserves_thought_signature_on_tool_replay():
|
||||
from agent.gemini_native_adapter import build_gemini_request
|
||||
|
||||
request = build_gemini_request(
|
||||
messages=[
|
||||
{"role": "system", "content": "Be helpful."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city": "Paris"}',
|
||||
},
|
||||
"extra_content": {
|
||||
"google": {"thought_signature": "sig-123"}
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
tools=[],
|
||||
tool_choice=None,
|
||||
)
|
||||
|
||||
parts = request["contents"][0]["parts"]
|
||||
assert parts[0]["functionCall"]["name"] == "get_weather"
|
||||
assert parts[0]["thoughtSignature"] == "sig-123"
|
||||
|
||||
|
||||
def test_build_native_request_uses_original_function_name_for_tool_result():
|
||||
from agent.gemini_native_adapter import build_gemini_request
|
||||
|
||||
request = build_gemini_request(
|
||||
messages=[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city": "Paris"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": '{"forecast": "sunny"}',
|
||||
},
|
||||
],
|
||||
tools=[],
|
||||
tool_choice=None,
|
||||
)
|
||||
|
||||
tool_response = request["contents"][1]["parts"][0]["functionResponse"]
|
||||
assert tool_response["name"] == "get_weather"
|
||||
|
||||
|
||||
def test_build_native_request_strips_json_schema_only_fields_from_tool_parameters():
|
||||
from agent.gemini_native_adapter import build_gemini_request
|
||||
|
||||
request = build_gemini_request(
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup_weather",
|
||||
"description": "Weather lookup",
|
||||
"parameters": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"$schema": "ignored",
|
||||
"description": "City name",
|
||||
}
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice=None,
|
||||
)
|
||||
|
||||
params = request["tools"][0]["functionDeclarations"][0]["parameters"]
|
||||
assert "$schema" not in params
|
||||
assert "additionalProperties" not in params
|
||||
assert params["type"] == "object"
|
||||
assert params["properties"]["city"] == {
|
||||
"type": "string",
|
||||
"description": "City name",
|
||||
}
|
||||
|
||||
|
||||
def test_translate_native_response_surfaces_reasoning_and_tool_calls():
|
||||
from agent.gemini_native_adapter import translate_gemini_response
|
||||
|
||||
payload = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{"thought": True, "text": "thinking..."},
|
||||
{"functionCall": {"name": "search", "args": {"q": "hermes"}}},
|
||||
]
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 10,
|
||||
"candidatesTokenCount": 5,
|
||||
"totalTokenCount": 15,
|
||||
},
|
||||
}
|
||||
|
||||
response = translate_gemini_response(payload, model="gemini-2.5-flash")
|
||||
choice = response.choices[0]
|
||||
assert choice.finish_reason == "tool_calls"
|
||||
assert choice.message.reasoning == "thinking..."
|
||||
assert choice.message.tool_calls[0].function.name == "search"
|
||||
assert json.loads(choice.message.tool_calls[0].function.arguments) == {"q": "hermes"}
|
||||
|
||||
|
||||
def test_native_client_uses_x_goog_api_key_and_native_models_endpoint(monkeypatch):
|
||||
from agent.gemini_native_adapter import GeminiNativeClient
|
||||
|
||||
recorded = {}
|
||||
|
||||
class DummyHTTP:
|
||||
def post(self, url, json=None, headers=None, timeout=None):
|
||||
recorded["url"] = url
|
||||
recorded["json"] = json
|
||||
recorded["headers"] = headers
|
||||
return DummyResponse(
|
||||
payload={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "hello"}]},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 1,
|
||||
"candidatesTokenCount": 1,
|
||||
"totalTokenCount": 2,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr("agent.gemini_native_adapter.httpx.Client", lambda *a, **k: DummyHTTP())
|
||||
|
||||
client = GeminiNativeClient(api_key="AIza-test", base_url="https://generativelanguage.googleapis.com/v1beta")
|
||||
response = client.chat.completions.create(
|
||||
model="gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
)
|
||||
|
||||
assert recorded["url"] == "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent"
|
||||
assert recorded["headers"]["x-goog-api-key"] == "AIza-test"
|
||||
assert "Authorization" not in recorded["headers"]
|
||||
assert response.choices[0].message.content == "hello"
|
||||
|
||||
|
||||
def test_native_http_error_keeps_status_and_retry_after():
|
||||
from agent.gemini_native_adapter import gemini_http_error
|
||||
|
||||
response = DummyResponse(
|
||||
status_code=429,
|
||||
headers={"Retry-After": "17"},
|
||||
payload={
|
||||
"error": {
|
||||
"code": 429,
|
||||
"message": "quota exhausted",
|
||||
"status": "RESOURCE_EXHAUSTED",
|
||||
"details": [
|
||||
{
|
||||
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
|
||||
"reason": "RESOURCE_EXHAUSTED",
|
||||
"metadata": {"service": "generativelanguage.googleapis.com"},
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
err = gemini_http_error(response)
|
||||
assert getattr(err, "status_code", None) == 429
|
||||
assert getattr(err, "retry_after", None) == 17.0
|
||||
assert "quota exhausted" in str(err)
|
||||
|
||||
|
||||
def test_native_client_accepts_injected_http_client():
|
||||
from agent.gemini_native_adapter import GeminiNativeClient
|
||||
|
||||
injected = SimpleNamespace(close=lambda: None)
|
||||
client = GeminiNativeClient(api_key="AIza-test", http_client=injected)
|
||||
assert client._http is injected
|
||||
|
||||
|
||||
def test_native_client_rejects_empty_api_key_with_actionable_message():
|
||||
"""Empty/whitespace api_key must raise at construction, not produce a cryptic
|
||||
Google GFE 'Error 400 (Bad Request)!!1' HTML page on the first request."""
|
||||
from agent.gemini_native_adapter import GeminiNativeClient
|
||||
|
||||
for bad in ("", " ", None):
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
GeminiNativeClient(api_key=bad) # type: ignore[arg-type]
|
||||
msg = str(excinfo.value)
|
||||
assert "GOOGLE_API_KEY" in msg and "GEMINI_API_KEY" in msg
|
||||
assert "aistudio.google.com" in msg
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_native_client_streams_without_requiring_async_iterator_from_sync_client():
|
||||
from agent.gemini_native_adapter import AsyncGeminiNativeClient
|
||||
|
||||
chunk = SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content="hi"), finish_reason=None)])
|
||||
sync_stream = iter([chunk])
|
||||
|
||||
def _advance(iterator):
|
||||
try:
|
||||
return False, next(iterator)
|
||||
except StopIteration:
|
||||
return True, None
|
||||
|
||||
sync_client = SimpleNamespace(
|
||||
api_key="AIza-test",
|
||||
base_url="https://generativelanguage.googleapis.com/v1beta",
|
||||
chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **kwargs: sync_stream)),
|
||||
_advance_stream_iterator=_advance,
|
||||
close=lambda: None,
|
||||
)
|
||||
|
||||
async_client = AsyncGeminiNativeClient(sync_client)
|
||||
stream = await async_client.chat.completions.create(stream=True)
|
||||
collected = []
|
||||
async for item in stream:
|
||||
collected.append(item)
|
||||
assert collected == [chunk]
|
||||
|
||||
|
||||
def test_stream_event_translation_emits_tool_call_delta_with_stable_index():
|
||||
from agent.gemini_native_adapter import translate_stream_event
|
||||
|
||||
tool_call_indices = {}
|
||||
event = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{"functionCall": {"name": "search", "args": {"q": "abc"}}}
|
||||
]
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
first = translate_stream_event(event, model="gemini-2.5-flash", tool_call_indices=tool_call_indices)
|
||||
second = translate_stream_event(event, model="gemini-2.5-flash", tool_call_indices=tool_call_indices)
|
||||
|
||||
assert first[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert second[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert first[0].choices[0].delta.tool_calls[0].id == second[0].choices[0].delta.tool_calls[0].id
|
||||
assert first[0].choices[0].delta.tool_calls[0].function.arguments == '{"q": "abc"}'
|
||||
assert second[0].choices[0].delta.tool_calls[0].function.arguments == ""
|
||||
assert first[-1].choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
def test_stream_event_translation_keeps_identical_calls_in_distinct_parts():
|
||||
from agent.gemini_native_adapter import translate_stream_event
|
||||
|
||||
event = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{"functionCall": {"name": "search", "args": {"q": "abc"}}},
|
||||
{"functionCall": {"name": "search", "args": {"q": "abc"}}},
|
||||
]
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
chunks = translate_stream_event(event, model="gemini-2.5-flash", tool_call_indices={})
|
||||
tool_chunks = [chunk for chunk in chunks if chunk.choices[0].delta.tool_calls]
|
||||
assert tool_chunks[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert tool_chunks[1].choices[0].delta.tool_calls[0].index == 1
|
||||
assert tool_chunks[0].choices[0].delta.tool_calls[0].id != tool_chunks[1].choices[0].delta.tool_calls[0].id
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Tests for agent.gemini_schema — OpenAI→Gemini tool parameter translation."""
|
||||
|
||||
from agent.gemini_schema import (
|
||||
sanitize_gemini_schema,
|
||||
sanitize_gemini_tool_parameters,
|
||||
)
|
||||
|
||||
|
||||
class TestSanitizeGeminiSchema:
|
||||
def test_strips_unknown_top_level_keys(self):
|
||||
"""$schema / additionalProperties etc. must not reach Gemini."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"additionalProperties": False,
|
||||
"properties": {"foo": {"type": "string"}},
|
||||
}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert "$schema" not in cleaned
|
||||
assert "additionalProperties" not in cleaned
|
||||
assert cleaned["type"] == "object"
|
||||
assert cleaned["properties"] == {"foo": {"type": "string"}}
|
||||
|
||||
def test_preserves_string_enums(self):
|
||||
"""String-valued enums are valid for Gemini and must pass through."""
|
||||
schema = {"type": "string", "enum": ["pending", "done", "cancelled"]}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert cleaned["type"] == "string"
|
||||
assert cleaned["enum"] == ["pending", "done", "cancelled"]
|
||||
|
||||
def test_drops_integer_enum_to_satisfy_gemini(self):
|
||||
"""Gemini rejects int-typed enums; the sanitizer must drop the enum.
|
||||
|
||||
Regression for the Discord tool's ``auto_archive_duration``:
|
||||
``{type: integer, enum: [60, 1440, 4320, 10080]}`` caused
|
||||
Gemini HTTP 400 INVALID_ARGUMENT
|
||||
"Invalid value ... (TYPE_STRING), 60" on every request that
|
||||
shipped the full tool catalog to generativelanguage.googleapis.com.
|
||||
"""
|
||||
schema = {
|
||||
"type": "integer",
|
||||
"enum": [60, 1440, 4320, 10080],
|
||||
"description": "Minutes (60, 1440, 4320, 10080).",
|
||||
}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert cleaned["type"] == "integer"
|
||||
assert "enum" not in cleaned
|
||||
# description must survive so the model still sees the allowed values
|
||||
assert cleaned["description"].startswith("Minutes")
|
||||
|
||||
def test_drops_number_enum(self):
|
||||
"""Same rule applies to ``type: number``."""
|
||||
schema = {"type": "number", "enum": [0.5, 1.0, 2.0]}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert cleaned["type"] == "number"
|
||||
assert "enum" not in cleaned
|
||||
|
||||
def test_drops_boolean_enum(self):
|
||||
"""And to ``type: boolean`` (Gemini rejects non-string entries)."""
|
||||
schema = {"type": "boolean", "enum": [True, False]}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert cleaned["type"] == "boolean"
|
||||
assert "enum" not in cleaned
|
||||
|
||||
def test_keeps_string_enum_even_when_numeric_values_coexist_as_strings(self):
|
||||
"""Stringified-numeric enums ARE valid for Gemini; don't drop them."""
|
||||
schema = {"type": "string", "enum": ["60", "1440", "4320", "10080"]}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert cleaned["enum"] == ["60", "1440", "4320", "10080"]
|
||||
|
||||
def test_drops_nested_integer_enum_inside_properties(self):
|
||||
"""The fix must apply recursively — the Discord case is nested."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"auto_archive_duration": {
|
||||
"type": "integer",
|
||||
"enum": [60, 1440, 4320, 10080],
|
||||
"description": "Thread archive duration in minutes.",
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["active", "archived"],
|
||||
},
|
||||
},
|
||||
}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
props = cleaned["properties"]
|
||||
# Integer enum is dropped...
|
||||
assert props["auto_archive_duration"]["type"] == "integer"
|
||||
assert "enum" not in props["auto_archive_duration"]
|
||||
# ...but the sibling string enum is preserved.
|
||||
assert props["status"]["enum"] == ["active", "archived"]
|
||||
|
||||
def test_drops_integer_enum_inside_array_items(self):
|
||||
"""Array item schemas recurse through ``items``."""
|
||||
schema = {
|
||||
"type": "array",
|
||||
"items": {"type": "integer", "enum": [1, 2, 3]},
|
||||
}
|
||||
cleaned = sanitize_gemini_schema(schema)
|
||||
assert cleaned["items"]["type"] == "integer"
|
||||
assert "enum" not in cleaned["items"]
|
||||
|
||||
def test_non_dict_input_returns_empty(self):
|
||||
assert sanitize_gemini_schema(None) == {}
|
||||
assert sanitize_gemini_schema("not a schema") == {}
|
||||
assert sanitize_gemini_schema([1, 2, 3]) == {}
|
||||
|
||||
|
||||
class TestSanitizeGeminiToolParameters:
|
||||
def test_empty_parameters_return_valid_object_schema(self):
|
||||
"""Gemini requires ``parameters`` to be a valid object schema."""
|
||||
cleaned = sanitize_gemini_tool_parameters({})
|
||||
assert cleaned == {"type": "object", "properties": {}}
|
||||
|
||||
def test_discord_create_thread_parameters_no_longer_trip_gemini(self):
|
||||
"""End-to-end regression: the exact shape that was rejected in prod."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "enum": ["create_thread"]},
|
||||
"auto_archive_duration": {
|
||||
"type": "integer",
|
||||
"enum": [60, 1440, 4320, 10080],
|
||||
"description": "Thread archive duration in minutes "
|
||||
"(create_thread, default 1440).",
|
||||
},
|
||||
},
|
||||
"required": ["action"],
|
||||
}
|
||||
cleaned = sanitize_gemini_tool_parameters(params)
|
||||
aad = cleaned["properties"]["auto_archive_duration"]
|
||||
# The field that triggered the Gemini 400 is gone.
|
||||
assert "enum" not in aad
|
||||
# Type + description survive so the model still knows what to send.
|
||||
assert aad["type"] == "integer"
|
||||
assert "1440" in aad["description"]
|
||||
# And the string-enum sibling is untouched.
|
||||
assert cleaned["properties"]["action"]["enum"] == ["create_thread"]
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Tests for agent.i18n -- catalog parity, fallback, language resolution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from agent import i18n
|
||||
|
||||
|
||||
LOCALES_DIR = Path(__file__).resolve().parents[2] / "locales"
|
||||
|
||||
|
||||
def _load_raw(lang: str) -> dict:
|
||||
with (LOCALES_DIR / f"{lang}.yaml").open("r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def _flatten(d, prefix="") -> dict:
|
||||
flat = {}
|
||||
for k, v in (d or {}).items():
|
||||
key = f"{prefix}.{k}" if prefix else k
|
||||
if isinstance(v, dict):
|
||||
flat.update(_flatten(v, key))
|
||||
else:
|
||||
flat[key] = v
|
||||
return flat
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog completeness -- this is the key invariant test. If someone adds a
|
||||
# new key to en.yaml they MUST add it to every other locale, else runtime
|
||||
# falls back to English for those users and defeats the feature.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_all_locales_exist():
|
||||
"""Every supported language must have a catalog file on disk."""
|
||||
for lang in i18n.SUPPORTED_LANGUAGES:
|
||||
assert (LOCALES_DIR / f"{lang}.yaml").is_file(), f"missing locales/{lang}.yaml"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lang", [l for l in i18n.SUPPORTED_LANGUAGES if l != "en"])
|
||||
def test_catalog_keys_match_english(lang: str):
|
||||
"""Every non-English catalog must have exactly the same key set as English."""
|
||||
en_keys = set(_flatten(_load_raw("en")).keys())
|
||||
lang_keys = set(_flatten(_load_raw(lang)).keys())
|
||||
missing = en_keys - lang_keys
|
||||
extra = lang_keys - en_keys
|
||||
assert not missing, f"{lang}.yaml missing keys: {sorted(missing)}"
|
||||
assert not extra, f"{lang}.yaml has keys not in en.yaml: {sorted(extra)}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lang", list(i18n.SUPPORTED_LANGUAGES))
|
||||
def test_catalog_placeholders_match_english(lang: str):
|
||||
"""Every translated value must use the same {placeholder} tokens as English.
|
||||
|
||||
A mistranslated placeholder (e.g. ``{description}`` typoed as ``{descricao}``)
|
||||
would either raise KeyError at runtime or silently drop the interpolated
|
||||
value. Pin parity at the test layer.
|
||||
"""
|
||||
import re
|
||||
placeholder_re = re.compile(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}")
|
||||
en_flat = _flatten(_load_raw("en"))
|
||||
lang_flat = _flatten(_load_raw(lang))
|
||||
for key, en_value in en_flat.items():
|
||||
en_placeholders = set(placeholder_re.findall(en_value))
|
||||
lang_value = lang_flat.get(key, "")
|
||||
lang_placeholders = set(placeholder_re.findall(lang_value))
|
||||
assert en_placeholders == lang_placeholders, (
|
||||
f"{lang}.yaml key={key!r}: placeholders {lang_placeholders} "
|
||||
f"don't match English {en_placeholders}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Language resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_normalize_lang_accepts_supported():
|
||||
assert i18n._normalize_lang("zh") == "zh"
|
||||
assert i18n._normalize_lang("EN") == "en"
|
||||
|
||||
|
||||
def test_normalize_lang_accepts_aliases():
|
||||
assert i18n._normalize_lang("chinese") == "zh"
|
||||
assert i18n._normalize_lang("zh-CN") == "zh"
|
||||
assert i18n._normalize_lang("Deutsch") == "de"
|
||||
assert i18n._normalize_lang("español") == "es"
|
||||
assert i18n._normalize_lang("jp") == "ja"
|
||||
assert i18n._normalize_lang("Ukrainian") == "uk"
|
||||
assert i18n._normalize_lang("uk-UA") == "uk"
|
||||
assert i18n._normalize_lang("ua") == "uk"
|
||||
assert i18n._normalize_lang("Turkish") == "tr"
|
||||
assert i18n._normalize_lang("tr-TR") == "tr"
|
||||
assert i18n._normalize_lang("türkçe") == "tr"
|
||||
|
||||
|
||||
def test_normalize_lang_unknown_falls_back():
|
||||
assert i18n._normalize_lang("klingon") == "en"
|
||||
assert i18n._normalize_lang("") == "en"
|
||||
assert i18n._normalize_lang(None) == "en"
|
||||
|
||||
|
||||
def test_env_var_override(monkeypatch):
|
||||
"""HERMES_LANGUAGE wins over config."""
|
||||
i18n.reset_language_cache()
|
||||
monkeypatch.setenv("HERMES_LANGUAGE", "ja")
|
||||
assert i18n.get_language() == "ja"
|
||||
|
||||
|
||||
def test_env_var_normalized(monkeypatch):
|
||||
i18n.reset_language_cache()
|
||||
monkeypatch.setenv("HERMES_LANGUAGE", "Chinese")
|
||||
assert i18n.get_language() == "zh"
|
||||
|
||||
|
||||
def test_default_when_nothing_set(monkeypatch):
|
||||
"""With no env var and no config override, falls back to English."""
|
||||
monkeypatch.delenv("HERMES_LANGUAGE", raising=False)
|
||||
# Force config lookup to return None -- patch the cached reader.
|
||||
i18n.reset_language_cache()
|
||||
monkeypatch.setattr(i18n, "_config_language_cached", lambda: None)
|
||||
assert i18n.get_language() == "en"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# t() semantics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_t_explicit_lang():
|
||||
assert i18n.t("approval.denied", lang="en").endswith("Denied")
|
||||
assert i18n.t("approval.denied", lang="zh").endswith("已拒绝")
|
||||
assert i18n.t("approval.denied", lang="uk").endswith("Відхилено")
|
||||
assert i18n.t("approval.denied", lang="tr").endswith("Reddedildi")
|
||||
|
||||
|
||||
def test_t_formats_placeholders():
|
||||
msg = i18n.t("gateway.draining", lang="en", count=3)
|
||||
assert "3" in msg
|
||||
|
||||
|
||||
def test_t_missing_key_returns_key():
|
||||
"""A missing key returns its own path -- ugly but never crashes."""
|
||||
result = i18n.t("nonexistent.key.path", lang="en")
|
||||
assert result == "nonexistent.key.path"
|
||||
|
||||
|
||||
def test_t_missing_key_in_non_english_falls_back_to_english(tmp_path, monkeypatch):
|
||||
"""If a key exists in English but not in the target locale, fall back."""
|
||||
# Stand up a fake incomplete locale under a temp locales dir.
|
||||
fake_locales = tmp_path / "locales"
|
||||
fake_locales.mkdir()
|
||||
(fake_locales / "en.yaml").write_text("foo: English Foo\n", encoding="utf-8")
|
||||
(fake_locales / "zh.yaml").write_text("# intentionally empty\n", encoding="utf-8")
|
||||
monkeypatch.setattr(i18n, "_locales_dir", lambda: fake_locales)
|
||||
i18n.reset_language_cache()
|
||||
try:
|
||||
assert i18n.t("foo", lang="zh") == "English Foo"
|
||||
finally:
|
||||
# Clear the cache on teardown so subsequent tests don't see the
|
||||
# fake "foo: English Foo" catalog instead of the real locales/*.yaml.
|
||||
i18n.reset_language_cache()
|
||||
|
||||
|
||||
def test_t_unknown_language_uses_english():
|
||||
"""Unknown lang codes normalize to English, not to a key-path fallback."""
|
||||
assert i18n.t("approval.denied", lang="klingon") == i18n.t("approval.denied", lang="en")
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Tests for agent/image_gen_registry.py — provider registration & active lookup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import image_gen_registry
|
||||
from agent.image_gen_provider import ImageGenProvider
|
||||
|
||||
|
||||
class _FakeProvider(ImageGenProvider):
|
||||
def __init__(self, name: str, available: bool = True):
|
||||
self._name = name
|
||||
self._available = available
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
def generate(self, prompt, aspect_ratio="landscape", **kw):
|
||||
return {"success": True, "image": f"{self._name}://{prompt}"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
image_gen_registry._reset_for_tests()
|
||||
yield
|
||||
image_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class TestRegisterProvider:
|
||||
def test_register_and_lookup(self):
|
||||
provider = _FakeProvider("fake")
|
||||
image_gen_registry.register_provider(provider)
|
||||
assert image_gen_registry.get_provider("fake") is provider
|
||||
|
||||
def test_rejects_non_provider(self):
|
||||
with pytest.raises(TypeError):
|
||||
image_gen_registry.register_provider("not a provider") # type: ignore[arg-type]
|
||||
|
||||
def test_rejects_empty_name(self):
|
||||
class Empty(ImageGenProvider):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return ""
|
||||
|
||||
def generate(self, prompt, aspect_ratio="landscape", **kw):
|
||||
return {}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
image_gen_registry.register_provider(Empty())
|
||||
|
||||
def test_reregister_overwrites(self):
|
||||
a = _FakeProvider("same")
|
||||
b = _FakeProvider("same")
|
||||
image_gen_registry.register_provider(a)
|
||||
image_gen_registry.register_provider(b)
|
||||
assert image_gen_registry.get_provider("same") is b
|
||||
|
||||
def test_list_is_sorted(self):
|
||||
image_gen_registry.register_provider(_FakeProvider("zeta"))
|
||||
image_gen_registry.register_provider(_FakeProvider("alpha"))
|
||||
names = [p.name for p in image_gen_registry.list_providers()]
|
||||
assert names == ["alpha", "zeta"]
|
||||
|
||||
|
||||
class TestGetActiveProvider:
|
||||
def test_single_provider_autoresolves(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
image_gen_registry.register_provider(_FakeProvider("solo"))
|
||||
active = image_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "solo"
|
||||
|
||||
def test_fal_preferred_on_multi_without_config(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
image_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
image_gen_registry.register_provider(_FakeProvider("openai"))
|
||||
active = image_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "fal"
|
||||
|
||||
def test_explicit_config_wins(self, tmp_path, monkeypatch):
|
||||
import yaml
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"image_gen": {"provider": "openai"}})
|
||||
)
|
||||
image_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
image_gen_registry.register_provider(_FakeProvider("openai"))
|
||||
active = image_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "openai"
|
||||
|
||||
def test_missing_configured_provider_falls_back(self, tmp_path, monkeypatch):
|
||||
import yaml
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"image_gen": {"provider": "replicate"}})
|
||||
)
|
||||
# Only FAL is registered — configured provider doesn't exist
|
||||
image_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
active = image_gen_registry.get_active_provider()
|
||||
# Falls back to FAL preference (legacy default) rather than None
|
||||
assert active is not None and active.name == "fal"
|
||||
|
||||
def test_none_when_empty(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
assert image_gen_registry.get_active_provider() is None
|
||||
@@ -0,0 +1,638 @@
|
||||
"""Tests for agent/image_routing.py — the per-turn image input mode decision."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
from agent.image_routing import (
|
||||
_coerce_capability_bool,
|
||||
_coerce_mode,
|
||||
_explicit_aux_vision_override,
|
||||
_lookup_supports_vision,
|
||||
_supports_vision_override,
|
||||
build_native_content_parts,
|
||||
decide_image_input_mode,
|
||||
extract_image_refs,
|
||||
)
|
||||
|
||||
|
||||
# ─── _coerce_mode ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCoerceMode:
|
||||
def test_valid_modes_pass_through(self):
|
||||
assert _coerce_mode("auto") == "auto"
|
||||
assert _coerce_mode("native") == "native"
|
||||
assert _coerce_mode("text") == "text"
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert _coerce_mode("NATIVE") == "native"
|
||||
assert _coerce_mode("Auto") == "auto"
|
||||
|
||||
def test_invalid_falls_back_to_auto(self):
|
||||
assert _coerce_mode("nonsense") == "auto"
|
||||
assert _coerce_mode("") == "auto"
|
||||
assert _coerce_mode(None) == "auto"
|
||||
assert _coerce_mode(42) == "auto"
|
||||
|
||||
def test_strips_whitespace(self):
|
||||
assert _coerce_mode(" native ") == "native"
|
||||
|
||||
|
||||
# ─── _explicit_aux_vision_override ───────────────────────────────────────────
|
||||
|
||||
|
||||
class TestExplicitAuxVisionOverride:
|
||||
def test_none_config(self):
|
||||
assert _explicit_aux_vision_override(None) is False
|
||||
|
||||
def test_empty_config(self):
|
||||
assert _explicit_aux_vision_override({}) is False
|
||||
|
||||
def test_default_auto_is_not_explicit(self):
|
||||
cfg = {"auxiliary": {"vision": {"provider": "auto", "model": "", "base_url": ""}}}
|
||||
assert _explicit_aux_vision_override(cfg) is False
|
||||
|
||||
def test_provider_set_is_explicit(self):
|
||||
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": ""}}}
|
||||
assert _explicit_aux_vision_override(cfg) is True
|
||||
|
||||
def test_model_set_is_explicit(self):
|
||||
cfg = {"auxiliary": {"vision": {"provider": "auto", "model": "google/gemini-2.5-flash"}}}
|
||||
assert _explicit_aux_vision_override(cfg) is True
|
||||
|
||||
def test_base_url_set_is_explicit(self):
|
||||
cfg = {"auxiliary": {"vision": {"provider": "auto", "base_url": "http://localhost:11434"}}}
|
||||
assert _explicit_aux_vision_override(cfg) is True
|
||||
|
||||
|
||||
# ─── decide_image_input_mode ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDecideImageInputMode:
|
||||
def test_explicit_native_overrides_everything(self):
|
||||
cfg = {"agent": {"image_input_mode": "native"}}
|
||||
# Non-vision model, aux-vision explicitly configured: native still wins.
|
||||
cfg["auxiliary"] = {"vision": {"provider": "openrouter", "model": "foo"}}
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=False):
|
||||
assert decide_image_input_mode("openrouter", "some-non-vision-model", cfg) == "native"
|
||||
|
||||
def test_explicit_text_overrides_everything(self):
|
||||
cfg = {"agent": {"image_input_mode": "text"}}
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
|
||||
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "text"
|
||||
|
||||
def test_auto_with_vision_capable_model(self):
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
|
||||
assert decide_image_input_mode("anthropic", "claude-sonnet-4", {}) == "native"
|
||||
|
||||
def test_auto_with_non_vision_model(self):
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=False):
|
||||
assert decide_image_input_mode("openrouter", "qwen/qwen3-235b", {}) == "text"
|
||||
|
||||
def test_auto_with_unknown_model(self):
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=None):
|
||||
assert decide_image_input_mode("openrouter", "brand-new-slug", {}) == "text"
|
||||
|
||||
def test_auto_respects_aux_vision_override_even_for_vision_model(self):
|
||||
"""If the user configured a dedicated vision backend, don't bypass it."""
|
||||
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"}}}
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
|
||||
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "text"
|
||||
|
||||
def test_none_config_is_auto(self):
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
|
||||
assert decide_image_input_mode("anthropic", "claude-sonnet-4", None) == "native"
|
||||
|
||||
def test_invalid_mode_coerces_to_auto(self):
|
||||
cfg = {"agent": {"image_input_mode": "weird-value"}}
|
||||
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
|
||||
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "native"
|
||||
|
||||
def test_auto_uses_text_for_text_only_modalities_even_with_attachment_flag(self):
|
||||
registry = {
|
||||
"xiaomi": {
|
||||
"models": {
|
||||
"mimo-v2.5-pro": {
|
||||
"attachment": True,
|
||||
"modalities": {"input": ["text"]},
|
||||
"tool_call": True,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=registry):
|
||||
assert decide_image_input_mode("xiaomi", "mimo-v2.5-pro", {}) == "text"
|
||||
|
||||
|
||||
# ─── _coerce_capability_bool ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCoerceCapabilityBool:
|
||||
def test_real_bool_passes_through(self):
|
||||
assert _coerce_capability_bool(True) is True
|
||||
assert _coerce_capability_bool(False) is False
|
||||
|
||||
def test_int_0_and_1(self):
|
||||
assert _coerce_capability_bool(1) is True
|
||||
assert _coerce_capability_bool(0) is False
|
||||
|
||||
def test_other_ints_return_none(self):
|
||||
assert _coerce_capability_bool(2) is None
|
||||
assert _coerce_capability_bool(-1) is None
|
||||
|
||||
def test_yaml_true_tokens(self):
|
||||
for s in ("true", "TRUE", "True", "yes", "on", "1", " true "):
|
||||
assert _coerce_capability_bool(s) is True
|
||||
|
||||
def test_yaml_false_tokens(self):
|
||||
for s in ("false", "FALSE", "False", "no", "off", "0", " false "):
|
||||
assert _coerce_capability_bool(s) is False
|
||||
|
||||
def test_quoted_false_does_not_silently_become_true(self):
|
||||
# Regression: bool("false") is True in Python. A user writing
|
||||
# supports_vision: "false" must NOT enable native vision routing.
|
||||
assert _coerce_capability_bool("false") is False
|
||||
|
||||
def test_unrecognised_strings_return_none(self):
|
||||
# None == fall through to models.dev, not a silent truthy.
|
||||
assert _coerce_capability_bool("maybe") is None
|
||||
assert _coerce_capability_bool("") is None
|
||||
assert _coerce_capability_bool("definitely") is None
|
||||
|
||||
def test_other_types_return_none(self):
|
||||
assert _coerce_capability_bool(None) is None
|
||||
assert _coerce_capability_bool([]) is None
|
||||
assert _coerce_capability_bool({}) is None
|
||||
assert _coerce_capability_bool(1.5) is None
|
||||
|
||||
|
||||
# ─── _supports_vision_override ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSupportsVisionOverride:
|
||||
def test_no_cfg_returns_none(self):
|
||||
assert _supports_vision_override(None, "custom", "my-llava") is None
|
||||
assert _supports_vision_override({}, "custom", "my-llava") is None
|
||||
|
||||
def test_top_level_shortcut_wins(self):
|
||||
cfg = {"model": {"supports_vision": True}}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is True
|
||||
|
||||
def test_top_level_false_propagates(self):
|
||||
cfg = {"model": {"supports_vision": False}}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is False
|
||||
|
||||
def test_per_provider_per_model_via_runtime_name(self):
|
||||
cfg = {
|
||||
"providers": {
|
||||
"custom": {"models": {"my-llava": {"supports_vision": True}}},
|
||||
},
|
||||
}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is True
|
||||
|
||||
def test_per_provider_per_model_via_config_name(self):
|
||||
# Named custom provider — runtime self.provider == "custom", config
|
||||
# holds the original name under model.provider.
|
||||
cfg = {
|
||||
"model": {"provider": "my-vllm"},
|
||||
"providers": {
|
||||
"my-vllm": {"models": {"my-llava": {"supports_vision": True}}},
|
||||
},
|
||||
}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is True
|
||||
|
||||
def test_quoted_false_string_in_yaml_does_not_enable(self):
|
||||
# Real-world: user writes supports_vision: "false" (quoted).
|
||||
cfg = {"model": {"supports_vision": "false"}}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is False
|
||||
|
||||
def test_unrecognised_value_falls_through(self):
|
||||
cfg = {"model": {"supports_vision": "maybe"}}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is None
|
||||
|
||||
def test_no_override_returns_none(self):
|
||||
cfg = {"model": {"default": "my-llava"}}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is None
|
||||
|
||||
def test_malformed_sections_are_ignored(self):
|
||||
# User accidentally wrote a string where a section was expected —
|
||||
# don't blow up, just fall through.
|
||||
cfg = {"model": "some-string", "providers": ["not-a-dict"]}
|
||||
assert _supports_vision_override(cfg, "custom", "my-llava") is None
|
||||
|
||||
|
||||
# ─── _lookup_supports_vision (override-aware) ────────────────────────────────
|
||||
|
||||
|
||||
class TestLookupSupportsVisionOverride:
|
||||
def test_config_override_short_circuits_models_dev(self):
|
||||
# Config says True, models.dev says None — config wins.
|
||||
cfg = {"model": {"supports_vision": True}}
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert _lookup_supports_vision("custom", "my-llava", cfg) is True
|
||||
|
||||
def test_config_override_false_beats_vision_capable_models_dev(self):
|
||||
# User explicitly disables vision on a models.dev-vision-capable model.
|
||||
fake_caps = type("Caps", (), {"supports_vision": True})()
|
||||
cfg = {"model": {"supports_vision": False}}
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=fake_caps):
|
||||
assert _lookup_supports_vision("anthropic", "claude-sonnet-4", cfg) is False
|
||||
|
||||
def test_no_override_falls_back_to_models_dev(self):
|
||||
fake_caps = type("Caps", (), {"supports_vision": True})()
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=fake_caps):
|
||||
assert _lookup_supports_vision("anthropic", "claude-sonnet-4", {}) is True
|
||||
|
||||
def test_no_override_no_models_dev_entry_returns_none(self):
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert _lookup_supports_vision("custom", "my-llava", {}) is None
|
||||
|
||||
def test_cfg_none_falls_back_to_models_dev(self):
|
||||
# Caller didn't pass cfg at all — old call sites must still work.
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert _lookup_supports_vision("openrouter", "x", None) is None
|
||||
|
||||
|
||||
# ─── decide_image_input_mode with auto + override ────────────────────────────
|
||||
|
||||
|
||||
class TestAutoModeRespectsOverride:
|
||||
def test_auto_native_for_custom_with_supports_vision_true(self):
|
||||
# The motivating bug: Qwen3.6 on local llama.cpp via provider=custom.
|
||||
# Without the override, auto falls back to text. With it, auto picks
|
||||
# native — no need to also set agent.image_input_mode: native.
|
||||
cfg = {"model": {"supports_vision": True}}
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert decide_image_input_mode("custom", "qwen3.6-35b", cfg) == "native"
|
||||
|
||||
def test_auto_text_for_custom_with_supports_vision_false(self):
|
||||
cfg = {"model": {"supports_vision": False}}
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert decide_image_input_mode("custom", "some-text-only", cfg) == "text"
|
||||
|
||||
def test_auto_text_for_custom_with_no_override(self):
|
||||
# Unchanged baseline: unknown custom model → text.
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert decide_image_input_mode("custom", "unknown", {}) == "text"
|
||||
|
||||
def test_explicit_aux_vision_override_still_wins(self):
|
||||
# If the user has configured a dedicated vision aux backend, respect
|
||||
# it even when supports_vision: true is also set.
|
||||
cfg = {
|
||||
"model": {"supports_vision": True},
|
||||
"auxiliary": {"vision": {"provider": "openrouter", "model": "gemini-2.5-pro"}},
|
||||
}
|
||||
with patch("agent.models_dev.get_model_capabilities", return_value=None):
|
||||
assert decide_image_input_mode("custom", "qwen3.6-35b", cfg) == "text"
|
||||
|
||||
|
||||
# ─── build_native_content_parts ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def _png_bytes() -> bytes:
|
||||
"""Return a tiny valid 1x1 transparent PNG."""
|
||||
return base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABpfZFQAAAAABJRU5ErkJggg=="
|
||||
)
|
||||
|
||||
|
||||
class TestBuildNativeContentParts:
|
||||
def test_text_then_image(self, tmp_path: Path):
|
||||
img = tmp_path / "cat.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
parts, skipped = build_native_content_parts("hello", [str(img)])
|
||||
assert skipped == []
|
||||
assert len(parts) == 2
|
||||
assert parts[0]["type"] == "text"
|
||||
# User caption is preserved and a per-image path hint is appended so
|
||||
# the model can use the local path as a string argument for tools
|
||||
# that take ``image_url: str`` (issue #18960).
|
||||
assert parts[0]["text"] == f"hello\n\n[Image attached at: {img}]"
|
||||
assert parts[1]["type"] == "image_url"
|
||||
assert parts[1]["image_url"]["url"].startswith("data:image/png;base64,")
|
||||
|
||||
def test_empty_text_inserts_default_prompt(self, tmp_path: Path):
|
||||
img = tmp_path / "cat.jpg"
|
||||
img.write_bytes(_png_bytes())
|
||||
parts, skipped = build_native_content_parts("", [str(img)])
|
||||
assert skipped == []
|
||||
# Even with empty user text, we insert a neutral prompt so the turn
|
||||
# isn't just pixels, and the path hint is appended after.
|
||||
assert parts[0]["type"] == "text"
|
||||
assert parts[0]["text"] == (
|
||||
f"What do you see in this image?\n\n[Image attached at: {img}]"
|
||||
)
|
||||
assert parts[1]["type"] == "image_url"
|
||||
|
||||
def test_missing_file_is_skipped(self, tmp_path: Path):
|
||||
parts, skipped = build_native_content_parts("hi", [str(tmp_path / "missing.png")])
|
||||
assert skipped == [str(tmp_path / "missing.png")]
|
||||
# Skipped paths are NOT advertised in the path hints — the model
|
||||
# would otherwise be told a non-existent file is attached.
|
||||
assert parts == [{"type": "text", "text": "hi"}]
|
||||
|
||||
def test_path_hint_appended(self, tmp_path: Path):
|
||||
"""The local path of each attached image is appended to the user
|
||||
text part so MCP/skill tools that take ``image_url: str`` can be
|
||||
invoked on the same image (issue #18960). Mirrors text-mode
|
||||
behaviour (`Runner._enrich_message_with_vision`).
|
||||
"""
|
||||
img = tmp_path / "scan.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
parts, _ = build_native_content_parts("attach this", [str(img)])
|
||||
text_part = next(p for p in parts if p.get("type") == "text")
|
||||
assert "[Image attached at:" in text_part["text"]
|
||||
assert str(img) in text_part["text"]
|
||||
# User caption is preserved verbatim ahead of the hint.
|
||||
assert text_part["text"].startswith("attach this")
|
||||
|
||||
def test_path_hint_one_per_attached_image(self, tmp_path: Path):
|
||||
"""Each successfully attached image gets its own path hint line;
|
||||
skipped images do NOT appear in the hints.
|
||||
"""
|
||||
good = tmp_path / "good.png"
|
||||
good.write_bytes(_png_bytes())
|
||||
missing = tmp_path / "missing.png" # never created
|
||||
parts, skipped = build_native_content_parts(
|
||||
"see attached", [str(good), str(missing)]
|
||||
)
|
||||
assert skipped == [str(missing)]
|
||||
text_part = next(p for p in parts if p.get("type") == "text")
|
||||
assert text_part["text"].count("[Image attached at:") == 1
|
||||
assert str(good) in text_part["text"]
|
||||
assert str(missing) not in text_part["text"]
|
||||
|
||||
def test_multiple_images(self, tmp_path: Path):
|
||||
img1 = tmp_path / "a.png"
|
||||
img2 = tmp_path / "b.png"
|
||||
img1.write_bytes(_png_bytes())
|
||||
img2.write_bytes(_png_bytes())
|
||||
parts, skipped = build_native_content_parts("compare these", [str(img1), str(img2)])
|
||||
assert skipped == []
|
||||
image_parts = [p for p in parts if p.get("type") == "image_url"]
|
||||
assert len(image_parts) == 2
|
||||
# Both paths surface in the text part, one per line.
|
||||
text_part = next(p for p in parts if p.get("type") == "text")
|
||||
assert text_part["text"].count("[Image attached at:") == 2
|
||||
assert str(img1) in text_part["text"]
|
||||
assert str(img2) in text_part["text"]
|
||||
|
||||
def test_mime_inference_jpg(self, tmp_path: Path):
|
||||
# Real JPEG bytes (SOI marker FF D8 FF): sniffing now wins over suffix.
|
||||
img = tmp_path / "photo.jpg"
|
||||
img.write_bytes(b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01" + b"\x00" * 32)
|
||||
parts, _ = build_native_content_parts("x", [str(img)])
|
||||
url = parts[1]["image_url"]["url"]
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
def test_mime_inference_webp(self, tmp_path: Path):
|
||||
# Real WEBP bytes (RIFF....WEBP): sniffing now wins over suffix.
|
||||
img = tmp_path / "pic.webp"
|
||||
img.write_bytes(b"RIFF\x24\x00\x00\x00WEBPVP8 " + b"\x00" * 32)
|
||||
parts, _ = build_native_content_parts("", [str(img)])
|
||||
url = parts[1]["image_url"]["url"]
|
||||
assert url.startswith("data:image/webp;base64,")
|
||||
|
||||
def test_mime_sniff_overrides_misleading_extension(self, tmp_path: Path):
|
||||
"""Discord-style bug: file is named .webp but contains PNG bytes.
|
||||
Anthropic rejects on MIME mismatch (HTTP 400) so we MUST sniff.
|
||||
Regression guard for the user-reported Discord PNG-as-WEBP failure.
|
||||
"""
|
||||
img = tmp_path / "discord_cached.webp"
|
||||
img.write_bytes(_png_bytes()) # bytes are PNG, suffix lies
|
||||
parts, _ = build_native_content_parts("", [str(img)])
|
||||
url = parts[1]["image_url"]["url"]
|
||||
assert url.startswith("data:image/png;base64,"), (
|
||||
f"Expected MIME sniffing to detect PNG bytes regardless of .webp suffix, got: {url[:60]}"
|
||||
)
|
||||
|
||||
|
||||
# ─── Oversize handling ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestLargeImageHandling:
|
||||
"""Large images attach at native size; shrink is handled reactively at
|
||||
retry time in ``run_agent._try_shrink_image_parts_in_messages`` rather
|
||||
than proactively here.
|
||||
"""
|
||||
|
||||
def test_large_image_passes_through_unchanged(self, tmp_path: Path):
|
||||
"""A multi-MB image is attached as-is — no resize, no skip."""
|
||||
from agent import image_routing as _ir
|
||||
|
||||
img = tmp_path / "medium.png"
|
||||
# 200 KB of real bytes; not huge but enough to verify no size gate fires.
|
||||
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"X" * 200_000)
|
||||
url = _ir._file_to_data_url(img)
|
||||
assert url is not None
|
||||
assert url.startswith("data:image/png;base64,")
|
||||
# Base64 expansion means output is ~4/3 of input, plus header.
|
||||
assert len(url) > 200_000
|
||||
|
||||
def test_missing_file_returns_none(self, tmp_path: Path):
|
||||
from agent import image_routing as _ir
|
||||
missing = tmp_path / "does_not_exist.png"
|
||||
assert _ir._file_to_data_url(missing) is None
|
||||
|
||||
def test_build_native_parts_no_provider_kwarg(self, tmp_path: Path):
|
||||
"""build_native_content_parts takes text + paths, no provider kwarg."""
|
||||
from agent import image_routing as _ir
|
||||
|
||||
img = tmp_path / "cat.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
parts, skipped = _ir.build_native_content_parts("hi", [str(img)])
|
||||
assert skipped == []
|
||||
assert len(parts) == 2
|
||||
assert parts[0]["type"] == "text"
|
||||
assert parts[1]["type"] == "image_url"
|
||||
|
||||
|
||||
# ─── extract_image_refs ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestExtractImageRefs:
|
||||
"""Scan task body / inbound text for image paths and URLs (kanban worker
|
||||
enrichment, issue raised May 2026)."""
|
||||
|
||||
def test_empty_or_none_returns_empty(self):
|
||||
assert extract_image_refs("") == ([], [])
|
||||
assert extract_image_refs(None) == ([], []) # type: ignore[arg-type]
|
||||
|
||||
def test_finds_absolute_path(self, tmp_path: Path):
|
||||
img = tmp_path / "screenshot.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
body = f"Look at {img} and tell me what's wrong."
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == [str(img)]
|
||||
assert urls == []
|
||||
|
||||
def test_finds_home_relative_path(self, tmp_path: Path, monkeypatch):
|
||||
# Simulate ~/foo.png by pointing HOME at tmp_path and creating the file
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
img = tmp_path / "foo.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
paths, urls = extract_image_refs("see ~/foo.png please")
|
||||
assert paths == [str(img)]
|
||||
assert urls == []
|
||||
|
||||
def test_skips_nonexistent_paths(self, tmp_path: Path):
|
||||
# Path-shaped but no file on disk → skipped.
|
||||
body = f"What's at {tmp_path}/never_created.png ?"
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == []
|
||||
assert urls == []
|
||||
|
||||
def test_finds_http_image_url(self):
|
||||
body = "Check out https://example.com/photos/cat.png — cute right?"
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == []
|
||||
assert urls == ["https://example.com/photos/cat.png"]
|
||||
|
||||
def test_finds_https_url_with_query_string(self):
|
||||
body = "Diagram: https://cdn.example.com/img.jpeg?size=large&v=2 here"
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert urls == ["https://cdn.example.com/img.jpeg?size=large&v=2"]
|
||||
|
||||
def test_url_trailing_punctuation_stripped(self):
|
||||
# Prose punctuation right after the URL must not be part of the URL.
|
||||
body = "See https://example.com/a.png."
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert urls == ["https://example.com/a.png"]
|
||||
|
||||
def test_ignores_non_image_urls(self):
|
||||
body = "See https://example.com/page.html and https://x.com/y.pdf"
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert urls == []
|
||||
|
||||
def test_dedupes_paths_and_urls(self, tmp_path: Path):
|
||||
img = tmp_path / "dup.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
body = (
|
||||
f"First {img} then again {img}. "
|
||||
"Also https://example.com/x.png and https://example.com/x.png again."
|
||||
)
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == [str(img)]
|
||||
assert urls == ["https://example.com/x.png"]
|
||||
|
||||
def test_ignores_paths_in_fenced_code_block(self, tmp_path: Path):
|
||||
img = tmp_path / "real.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
body = (
|
||||
"Outside the block, attach this:\n"
|
||||
f"{img}\n"
|
||||
"But not these examples:\n"
|
||||
"```\n"
|
||||
f"some_other_image: /tmp/example.png\n"
|
||||
f"url: https://example.com/example.png\n"
|
||||
"```\n"
|
||||
)
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == [str(img)]
|
||||
assert urls == []
|
||||
|
||||
def test_ignores_paths_in_inline_code(self, tmp_path: Path):
|
||||
img = tmp_path / "real.jpg"
|
||||
img.write_bytes(_png_bytes())
|
||||
body = (
|
||||
f"Attach {img}, but ignore the example "
|
||||
"`https://example.com/skip.png` in backticks."
|
||||
)
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == [str(img)]
|
||||
assert urls == []
|
||||
|
||||
def test_does_not_match_paths_inside_urls(self, tmp_path: Path):
|
||||
# The lookbehind in the regex prevents matching the path-portion of
|
||||
# a URL as a local path. Only the URL should be detected.
|
||||
body = "Just the URL: https://example.com/some/dir/image.png"
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == []
|
||||
assert urls == ["https://example.com/some/dir/image.png"]
|
||||
|
||||
def test_mixed_paths_and_urls(self, tmp_path: Path):
|
||||
img = tmp_path / "local.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
body = (
|
||||
f"Compare local {img} against the design at "
|
||||
"https://example.com/design/v2.png — does it match?"
|
||||
)
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == [str(img)]
|
||||
assert urls == ["https://example.com/design/v2.png"]
|
||||
|
||||
def test_case_insensitive_extension(self, tmp_path: Path):
|
||||
img = tmp_path / "shouty.PNG"
|
||||
img.write_bytes(_png_bytes())
|
||||
body = f"see {img}"
|
||||
paths, urls = extract_image_refs(body)
|
||||
assert paths == [str(img)]
|
||||
|
||||
|
||||
# ─── build_native_content_parts with URLs ────────────────────────────────────
|
||||
|
||||
|
||||
class TestBuildNativeContentPartsURLs:
|
||||
"""URL pass-through support added so kanban task bodies (and other
|
||||
inbound surfaces) can route remote image URLs straight to the model."""
|
||||
|
||||
def test_url_only_no_local_paths(self):
|
||||
parts, skipped = build_native_content_parts(
|
||||
"what is this?",
|
||||
[],
|
||||
image_urls=["https://example.com/diagram.png"],
|
||||
)
|
||||
assert skipped == []
|
||||
assert len(parts) == 2
|
||||
assert parts[0]["type"] == "text"
|
||||
assert "[Image attached: https://example.com/diagram.png]" in parts[0]["text"]
|
||||
assert parts[0]["text"].startswith("what is this?")
|
||||
assert parts[1] == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/diagram.png"},
|
||||
}
|
||||
|
||||
def test_mixed_path_and_url(self, tmp_path: Path):
|
||||
img = tmp_path / "local.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
parts, skipped = build_native_content_parts(
|
||||
"compare these",
|
||||
[str(img)],
|
||||
image_urls=["https://example.com/remote.jpg"],
|
||||
)
|
||||
assert skipped == []
|
||||
# 1 text + 2 image parts (local data URL first, then remote URL).
|
||||
image_parts = [p for p in parts if p.get("type") == "image_url"]
|
||||
assert len(image_parts) == 2
|
||||
assert image_parts[0]["image_url"]["url"].startswith("data:image/png;base64,")
|
||||
assert image_parts[1]["image_url"]["url"] == "https://example.com/remote.jpg"
|
||||
text = parts[0]["text"]
|
||||
assert "[Image attached at:" in text
|
||||
assert "[Image attached: https://example.com/remote.jpg]" in text
|
||||
|
||||
def test_empty_url_list_is_no_op(self, tmp_path: Path):
|
||||
img = tmp_path / "x.png"
|
||||
img.write_bytes(_png_bytes())
|
||||
# image_urls=[] should behave the same as not passing it at all.
|
||||
parts_no_urls, _ = build_native_content_parts("hi", [str(img)])
|
||||
parts_empty_urls, _ = build_native_content_parts("hi", [str(img)], image_urls=[])
|
||||
assert parts_no_urls == parts_empty_urls
|
||||
|
||||
def test_blank_url_strings_are_dropped(self):
|
||||
parts, _ = build_native_content_parts(
|
||||
"x", [], image_urls=["", " ", "https://example.com/a.png"]
|
||||
)
|
||||
image_parts = [p for p in parts if p.get("type") == "image_url"]
|
||||
assert len(image_parts) == 1
|
||||
assert image_parts[0]["image_url"]["url"] == "https://example.com/a.png"
|
||||
|
||||
def test_url_only_inserts_default_prompt_when_text_empty(self):
|
||||
parts, _ = build_native_content_parts(
|
||||
"", [], image_urls=["https://example.com/a.png"]
|
||||
)
|
||||
assert parts[0]["type"] == "text"
|
||||
assert parts[0]["text"].startswith("What do you see in this image?")
|
||||
@@ -0,0 +1,730 @@
|
||||
"""Tests for agent/insights.py — InsightsEngine analytics and reporting."""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB
|
||||
from agent.insights import (
|
||||
InsightsEngine,
|
||||
_estimate_cost,
|
||||
_format_duration,
|
||||
_bar_chart,
|
||||
_has_known_pricing,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path):
|
||||
"""Create a SessionDB with a temp database file."""
|
||||
db_path = tmp_path / "test_insights.db"
|
||||
session_db = SessionDB(db_path=db_path)
|
||||
yield session_db
|
||||
session_db.close()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def populated_db(db):
|
||||
"""Create a DB with realistic session data for insights testing."""
|
||||
now = time.time()
|
||||
day = 86400
|
||||
|
||||
# Session 1: CLI, claude-sonnet, ended, 2 days ago
|
||||
db.create_session(
|
||||
session_id="s1", source="cli",
|
||||
model="anthropic/claude-sonnet-4-20250514", user_id="user1",
|
||||
)
|
||||
# Backdate the started_at
|
||||
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's1'", (now - 2 * day,))
|
||||
db.end_session("s1", end_reason="user_exit")
|
||||
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's1'", (now - 2 * day + 3600,))
|
||||
db.update_token_counts("s1", input_tokens=50000, output_tokens=15000)
|
||||
db.append_message("s1", role="user", content="Hello, help me fix a bug")
|
||||
db.append_message("s1", role="assistant", content="Sure, let me look into that.")
|
||||
db.append_message("s1", role="assistant", content="Let me search the files.",
|
||||
tool_calls=[{"function": {"name": "search_files"}}])
|
||||
db.append_message("s1", role="tool", content="Found 3 matches", tool_name="search_files")
|
||||
db.append_message("s1", role="assistant", content="Let me read the file.",
|
||||
tool_calls=[{"function": {"name": "read_file"}}])
|
||||
db.append_message("s1", role="tool", content="file contents...", tool_name="read_file")
|
||||
db.append_message("s1", role="assistant", content="I found the bug. Let me fix it.",
|
||||
tool_calls=[{"function": {"name": "patch"}}])
|
||||
db.append_message("s1", role="tool", content="patched successfully", tool_name="patch")
|
||||
db.append_message(
|
||||
"s1",
|
||||
role="assistant",
|
||||
content="Let me load the PR workflow skill.",
|
||||
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}],
|
||||
)
|
||||
db.append_message("s1", role="user", content="Thanks!")
|
||||
db.append_message("s1", role="assistant", content="You're welcome!")
|
||||
|
||||
# Session 2: Telegram, gpt-4o, ended, 5 days ago
|
||||
db.create_session(
|
||||
session_id="s2", source="telegram",
|
||||
model="gpt-4o", user_id="user1",
|
||||
)
|
||||
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's2'", (now - 5 * day,))
|
||||
db.end_session("s2", end_reason="timeout")
|
||||
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's2'", (now - 5 * day + 1800,))
|
||||
db.update_token_counts("s2", input_tokens=20000, output_tokens=8000)
|
||||
db.append_message("s2", role="user", content="Search the web for something")
|
||||
db.append_message("s2", role="assistant", content="Searching...",
|
||||
tool_calls=[{"function": {"name": "web_search"}}])
|
||||
db.append_message("s2", role="tool", content="results...", tool_name="web_search")
|
||||
db.append_message("s2", role="assistant", content="Here's what I found")
|
||||
|
||||
# Session 3: CLI, deepseek-chat, ended, 10 days ago
|
||||
db.create_session(
|
||||
session_id="s3", source="cli",
|
||||
model="deepseek-chat", user_id="user1",
|
||||
)
|
||||
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's3'", (now - 10 * day,))
|
||||
db.end_session("s3", end_reason="user_exit")
|
||||
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's3'", (now - 10 * day + 7200,))
|
||||
db.update_token_counts("s3", input_tokens=100000, output_tokens=40000)
|
||||
db.append_message("s3", role="user", content="Run this terminal command")
|
||||
db.append_message("s3", role="assistant", content="Running...",
|
||||
tool_calls=[{"function": {"name": "terminal"}}])
|
||||
db.append_message("s3", role="tool", content="output...", tool_name="terminal")
|
||||
db.append_message("s3", role="assistant", content="Let me run another",
|
||||
tool_calls=[{"function": {"name": "terminal"}}])
|
||||
db.append_message("s3", role="tool", content="more output...", tool_name="terminal")
|
||||
db.append_message("s3", role="assistant", content="And search files",
|
||||
tool_calls=[{"function": {"name": "search_files"}}])
|
||||
db.append_message("s3", role="tool", content="found stuff", tool_name="search_files")
|
||||
db.append_message(
|
||||
"s3",
|
||||
role="assistant",
|
||||
content="Load the debugging skill.",
|
||||
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"systematic-debugging"}'}}],
|
||||
)
|
||||
|
||||
# Session 4: Discord, same model as s1, ended, 1 day ago
|
||||
db.create_session(
|
||||
session_id="s4", source="discord",
|
||||
model="anthropic/claude-sonnet-4-20250514", user_id="user2",
|
||||
)
|
||||
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's4'", (now - 1 * day,))
|
||||
db.end_session("s4", end_reason="user_exit")
|
||||
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's4'", (now - 1 * day + 900,))
|
||||
db.update_token_counts("s4", input_tokens=10000, output_tokens=5000)
|
||||
db.append_message("s4", role="user", content="Quick question")
|
||||
db.append_message("s4", role="assistant", content="Sure, go ahead")
|
||||
db.append_message(
|
||||
"s4",
|
||||
role="assistant",
|
||||
content="Load and update GitHub skills.",
|
||||
tool_calls=[
|
||||
{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}},
|
||||
{"function": {"name": "skill_manage", "arguments": '{"name":"github-code-review"}'}},
|
||||
],
|
||||
)
|
||||
|
||||
# Session 5: Old session, 45 days ago (should be excluded from 30-day window)
|
||||
db.create_session(
|
||||
session_id="s_old", source="cli",
|
||||
model="gpt-4o-mini", user_id="user1",
|
||||
)
|
||||
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's_old'", (now - 45 * day,))
|
||||
db.end_session("s_old", end_reason="user_exit")
|
||||
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's_old'", (now - 45 * day + 600,))
|
||||
db.update_token_counts("s_old", input_tokens=5000, output_tokens=2000)
|
||||
db.append_message("s_old", role="user", content="old message")
|
||||
db.append_message("s_old", role="assistant", content="old reply")
|
||||
|
||||
db._conn.commit()
|
||||
return db
|
||||
|
||||
|
||||
class TestHasKnownPricing:
|
||||
def test_known_commercial_model(self):
|
||||
assert _has_known_pricing("gpt-4o", provider="openai") is True
|
||||
assert _has_known_pricing("anthropic/claude-sonnet-4-20250514") is True
|
||||
assert _has_known_pricing("gpt-4.1", provider="openai") is True
|
||||
|
||||
def test_unknown_custom_model(self):
|
||||
assert _has_known_pricing("FP16_Hermes_4.5") is False
|
||||
assert _has_known_pricing("my-custom-model") is False
|
||||
assert _has_known_pricing("glm-5") is False
|
||||
assert _has_known_pricing("") is False
|
||||
assert _has_known_pricing(None) is False
|
||||
|
||||
def test_heuristic_matched_models_are_not_considered_known(self):
|
||||
assert _has_known_pricing("some-opus-model") is False
|
||||
assert _has_known_pricing("future-sonnet-v2") is False
|
||||
|
||||
|
||||
class TestEstimateCost:
|
||||
def test_basic_cost(self):
|
||||
cost, status = _estimate_cost(
|
||||
"anthropic/claude-sonnet-4-20250514",
|
||||
1_000_000,
|
||||
1_000_000,
|
||||
provider="anthropic",
|
||||
)
|
||||
assert status == "estimated"
|
||||
assert cost == pytest.approx(18.0, abs=0.01)
|
||||
|
||||
def test_zero_tokens(self):
|
||||
cost, status = _estimate_cost("gpt-4o", 0, 0, provider="openai")
|
||||
assert status == "estimated"
|
||||
assert cost == 0.0
|
||||
|
||||
def test_cache_aware_usage(self):
|
||||
cost, status = _estimate_cost(
|
||||
"anthropic/claude-sonnet-4-20250514",
|
||||
1000,
|
||||
500,
|
||||
cache_read_tokens=2000,
|
||||
cache_write_tokens=400,
|
||||
provider="anthropic",
|
||||
)
|
||||
assert status == "estimated"
|
||||
expected = (1000 * 3.0 + 500 * 15.0 + 2000 * 0.30 + 400 * 3.75) / 1_000_000
|
||||
assert cost == pytest.approx(expected, abs=0.0001)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Format helpers
|
||||
# =========================================================================
|
||||
|
||||
class TestFormatDuration:
|
||||
def test_seconds(self):
|
||||
assert _format_duration(45) == "45s"
|
||||
|
||||
def test_minutes(self):
|
||||
assert _format_duration(300) == "5m"
|
||||
|
||||
def test_hours_with_minutes(self):
|
||||
result = _format_duration(5400) # 1.5 hours
|
||||
assert result == "1h 30m"
|
||||
|
||||
def test_exact_hours(self):
|
||||
assert _format_duration(7200) == "2h"
|
||||
|
||||
def test_days(self):
|
||||
result = _format_duration(172800) # 2 days
|
||||
assert result == "2.0d"
|
||||
|
||||
|
||||
class TestBarChart:
|
||||
def test_basic_bars(self):
|
||||
bars = _bar_chart([10, 5, 0, 20], max_width=10)
|
||||
assert len(bars) == 4
|
||||
assert len(bars[3]) == 10 # max value gets full width
|
||||
assert len(bars[0]) == 5 # half of max
|
||||
assert bars[2] == "" # zero gets empty
|
||||
|
||||
def test_empty_values(self):
|
||||
bars = _bar_chart([], max_width=10)
|
||||
assert bars == []
|
||||
|
||||
def test_all_zeros(self):
|
||||
bars = _bar_chart([0, 0, 0], max_width=10)
|
||||
assert all(b == "" for b in bars)
|
||||
|
||||
def test_single_value(self):
|
||||
bars = _bar_chart([5], max_width=10)
|
||||
assert len(bars) == 1
|
||||
assert len(bars[0]) == 10
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# InsightsEngine — empty DB
|
||||
# =========================================================================
|
||||
|
||||
class TestInsightsEmpty:
|
||||
def test_empty_db_returns_empty_report(self, db):
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["empty"] is True
|
||||
assert report["overview"] == {}
|
||||
|
||||
def test_empty_db_terminal_format(self, db):
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_terminal(report)
|
||||
assert "No sessions found" in text
|
||||
|
||||
def test_empty_db_gateway_format(self, db):
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_gateway(report)
|
||||
assert "No sessions found" in text
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# InsightsEngine — populated DB
|
||||
# =========================================================================
|
||||
|
||||
class TestInsightsPopulated:
|
||||
def test_generate_returns_all_sections(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
|
||||
assert report["empty"] is False
|
||||
assert "overview" in report
|
||||
assert "models" in report
|
||||
assert "platforms" in report
|
||||
assert "tools" in report
|
||||
assert "activity" in report
|
||||
assert "top_sessions" in report
|
||||
|
||||
def test_overview_session_count(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
overview = report["overview"]
|
||||
|
||||
# s1, s2, s3, s4 are within 30 days; s_old is 45 days ago
|
||||
assert overview["total_sessions"] == 4
|
||||
|
||||
def test_overview_token_totals(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
overview = report["overview"]
|
||||
|
||||
expected_input = 50000 + 20000 + 100000 + 10000
|
||||
expected_output = 15000 + 8000 + 40000 + 5000
|
||||
assert overview["total_input_tokens"] == expected_input
|
||||
assert overview["total_output_tokens"] == expected_output
|
||||
assert overview["total_tokens"] == expected_input + expected_output
|
||||
|
||||
def test_overview_cost_positive(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["overview"]["estimated_cost"] > 0
|
||||
|
||||
def test_overview_duration_stats(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
overview = report["overview"]
|
||||
|
||||
# All 4 sessions have durations
|
||||
assert overview["total_hours"] > 0
|
||||
assert overview["avg_session_duration"] > 0
|
||||
|
||||
def test_model_breakdown(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
models = report["models"]
|
||||
|
||||
# Should have 3 distinct models (claude-sonnet x2, gpt-4o, deepseek-chat)
|
||||
model_names = [m["model"] for m in models]
|
||||
assert "claude-sonnet-4-20250514" in model_names
|
||||
assert "gpt-4o" in model_names
|
||||
assert "deepseek-chat" in model_names
|
||||
|
||||
# Claude-sonnet has 2 sessions (s1 + s4)
|
||||
claude = next(m for m in models if "claude-sonnet" in m["model"])
|
||||
assert claude["sessions"] == 2
|
||||
|
||||
def test_platform_breakdown(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
platforms = report["platforms"]
|
||||
|
||||
platform_names = [p["platform"] for p in platforms]
|
||||
assert "cli" in platform_names
|
||||
assert "telegram" in platform_names
|
||||
assert "discord" in platform_names
|
||||
|
||||
cli = next(p for p in platforms if p["platform"] == "cli")
|
||||
assert cli["sessions"] == 2 # s1 + s3
|
||||
|
||||
def test_tool_breakdown(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
tools = report["tools"]
|
||||
|
||||
tool_names = [t["tool"] for t in tools]
|
||||
assert "terminal" in tool_names
|
||||
assert "search_files" in tool_names
|
||||
assert "read_file" in tool_names
|
||||
assert "patch" in tool_names
|
||||
assert "web_search" in tool_names
|
||||
|
||||
# terminal was used 2x in s3
|
||||
terminal = next(t for t in tools if t["tool"] == "terminal")
|
||||
assert terminal["count"] == 2
|
||||
|
||||
# Percentages should sum to ~100%
|
||||
total_pct = sum(t["percentage"] for t in tools)
|
||||
assert total_pct == pytest.approx(100.0, abs=0.1)
|
||||
|
||||
def test_skill_breakdown(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
skills = report["skills"]
|
||||
|
||||
assert skills["summary"]["distinct_skills_used"] == 3
|
||||
assert skills["summary"]["total_skill_loads"] == 3
|
||||
assert skills["summary"]["total_skill_edits"] == 1
|
||||
assert skills["summary"]["total_skill_actions"] == 4
|
||||
|
||||
top_skill = skills["top_skills"][0]
|
||||
assert top_skill["skill"] == "github-pr-workflow"
|
||||
assert top_skill["view_count"] == 2
|
||||
assert top_skill["manage_count"] == 0
|
||||
assert top_skill["total_count"] == 2
|
||||
assert top_skill["last_used_at"] is not None
|
||||
|
||||
def test_skill_breakdown_respects_days_filter(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=3)
|
||||
skills = report["skills"]
|
||||
|
||||
assert skills["summary"]["distinct_skills_used"] == 2
|
||||
assert skills["summary"]["total_skill_loads"] == 2
|
||||
assert skills["summary"]["total_skill_edits"] == 1
|
||||
|
||||
skill_names = [s["skill"] for s in skills["top_skills"]]
|
||||
assert "systematic-debugging" not in skill_names
|
||||
|
||||
def test_activity_patterns(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
activity = report["activity"]
|
||||
|
||||
assert len(activity["by_day"]) == 7
|
||||
assert len(activity["by_hour"]) == 24
|
||||
assert activity["active_days"] >= 1
|
||||
assert activity["busiest_day"] is not None
|
||||
assert activity["busiest_hour"] is not None
|
||||
|
||||
def test_top_sessions(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
top = report["top_sessions"]
|
||||
|
||||
labels = [t["label"] for t in top]
|
||||
assert "Longest session" in labels
|
||||
assert "Most messages" in labels
|
||||
assert "Most tokens" in labels
|
||||
assert "Most tool calls" in labels
|
||||
|
||||
def test_source_filter_cli(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30, source="cli")
|
||||
|
||||
assert report["overview"]["total_sessions"] == 2 # s1, s3
|
||||
|
||||
def test_source_filter_telegram(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30, source="telegram")
|
||||
|
||||
assert report["overview"]["total_sessions"] == 1 # s2
|
||||
|
||||
def test_source_filter_nonexistent(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30, source="slack")
|
||||
|
||||
assert report["empty"] is True
|
||||
|
||||
def test_days_filter_short(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=3)
|
||||
|
||||
# Only s1 (2 days ago) and s4 (1 day ago) should be included
|
||||
assert report["overview"]["total_sessions"] == 2
|
||||
|
||||
def test_days_filter_long(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=60)
|
||||
|
||||
# All 5 sessions should be included
|
||||
assert report["overview"]["total_sessions"] == 5
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Formatting
|
||||
# =========================================================================
|
||||
|
||||
class TestTerminalFormatting:
|
||||
def test_terminal_format_has_sections(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_terminal(report)
|
||||
|
||||
assert "Hermes Insights" in text
|
||||
assert "Overview" in text
|
||||
assert "Models Used" in text
|
||||
assert "Top Tools" in text
|
||||
assert "Top Skills" in text
|
||||
assert "Activity Patterns" in text
|
||||
assert "Notable Sessions" in text
|
||||
|
||||
def test_terminal_format_shows_tokens(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_terminal(report)
|
||||
|
||||
assert "Input tokens" in text
|
||||
assert "Output tokens" in text
|
||||
# Cost and cache metrics are intentionally hidden (pricing was unreliable).
|
||||
assert "Est. cost" not in text
|
||||
assert "Cache read" not in text
|
||||
assert "Cache write" not in text
|
||||
|
||||
def test_terminal_format_shows_platforms(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_terminal(report)
|
||||
|
||||
# Multi-platform, so Platforms section should show
|
||||
assert "Platforms" in text
|
||||
assert "cli" in text
|
||||
assert "telegram" in text
|
||||
|
||||
def test_terminal_format_shows_bar_chart(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_terminal(report)
|
||||
|
||||
assert "█" in text # Bar chart characters
|
||||
|
||||
def test_terminal_format_hides_cost_for_custom_models(self, db):
|
||||
"""Cost display is hidden entirely — custom models no longer show 'N/A' either."""
|
||||
db.create_session(session_id="s1", source="cli", model="my-custom-model")
|
||||
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_terminal(report)
|
||||
|
||||
assert "N/A" not in text
|
||||
assert "custom/self-hosted" not in text
|
||||
assert "Cost" not in text
|
||||
|
||||
|
||||
class TestGatewayFormatting:
|
||||
def test_gateway_format_is_shorter(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
terminal_text = engine.format_terminal(report)
|
||||
gateway_text = engine.format_gateway(report)
|
||||
|
||||
assert len(gateway_text) < len(terminal_text)
|
||||
|
||||
def test_gateway_format_has_bold(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_gateway(report)
|
||||
|
||||
assert "**" in text # Markdown bold
|
||||
|
||||
def test_gateway_format_hides_cost(self, populated_db):
|
||||
"""Gateway format omits dollar figures and internal cache details."""
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_gateway(report)
|
||||
|
||||
assert "$" not in text
|
||||
assert "cache" not in text.lower()
|
||||
|
||||
def test_gateway_format_shows_models(self, populated_db):
|
||||
engine = InsightsEngine(populated_db)
|
||||
report = engine.generate(days=30)
|
||||
text = engine.format_gateway(report)
|
||||
|
||||
assert "Models" in text
|
||||
assert "sessions" in text
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Edge cases
|
||||
# =========================================================================
|
||||
|
||||
class TestEdgeCases:
|
||||
def test_session_with_no_tokens(self, db):
|
||||
"""Sessions with zero tokens should not crash."""
|
||||
db.create_session(session_id="s1", source="cli", model="test-model")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["empty"] is False
|
||||
assert report["overview"]["total_tokens"] == 0
|
||||
assert report["overview"]["estimated_cost"] == 0.0
|
||||
|
||||
def test_session_with_no_end_time(self, db):
|
||||
"""Active (non-ended) sessions should be included but duration = 0."""
|
||||
db.create_session(session_id="s1", source="cli", model="test-model")
|
||||
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
# Session included
|
||||
assert report["overview"]["total_sessions"] == 1
|
||||
assert report["overview"]["total_tokens"] == 1500
|
||||
# But no duration stats (session not ended)
|
||||
assert report["overview"]["total_hours"] == 0
|
||||
|
||||
def test_session_with_no_model(self, db):
|
||||
"""Sessions with NULL model should not crash."""
|
||||
db.create_session(session_id="s1", source="cli")
|
||||
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["empty"] is False
|
||||
|
||||
models = report["models"]
|
||||
assert len(models) == 1
|
||||
assert models[0]["model"] == "unknown"
|
||||
assert models[0]["has_pricing"] is False
|
||||
|
||||
def test_custom_model_shows_zero_cost(self, db):
|
||||
"""Custom/self-hosted models should show $0 cost, not fake estimates."""
|
||||
db.create_session(session_id="s1", source="cli", model="FP16_Hermes_4.5")
|
||||
db.update_token_counts("s1", input_tokens=100000, output_tokens=50000)
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["overview"]["estimated_cost"] == 0.0
|
||||
assert "FP16_Hermes_4.5" in report["overview"]["models_without_pricing"]
|
||||
|
||||
models = report["models"]
|
||||
custom = next(m for m in models if m["model"] == "FP16_Hermes_4.5")
|
||||
assert custom["cost"] == 0.0
|
||||
assert custom["has_pricing"] is False
|
||||
|
||||
def test_tool_usage_from_tool_calls_json(self, db):
|
||||
"""Tool usage should be extracted from tool_calls JSON when tool_name is NULL."""
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
# Assistant message with tool_calls (this is what CLI produces)
|
||||
db.append_message("s1", role="assistant", content="Let me search",
|
||||
tool_calls=[{"id": "call_1", "type": "function",
|
||||
"function": {"name": "search_files", "arguments": "{}"}}])
|
||||
# Tool response WITHOUT tool_name (this is the CLI bug)
|
||||
db.append_message("s1", role="tool", content="found results",
|
||||
tool_call_id="call_1")
|
||||
db.append_message("s1", role="assistant", content="Now reading",
|
||||
tool_calls=[{"id": "call_2", "type": "function",
|
||||
"function": {"name": "read_file", "arguments": "{}"}}])
|
||||
db.append_message("s1", role="tool", content="file content",
|
||||
tool_call_id="call_2")
|
||||
db.append_message("s1", role="assistant", content="And searching again",
|
||||
tool_calls=[{"id": "call_3", "type": "function",
|
||||
"function": {"name": "search_files", "arguments": "{}"}}])
|
||||
db.append_message("s1", role="tool", content="more results",
|
||||
tool_call_id="call_3")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
tools = report["tools"]
|
||||
|
||||
# Should find tools from tool_calls JSON even though tool_name is NULL
|
||||
tool_names = [t["tool"] for t in tools]
|
||||
assert "search_files" in tool_names
|
||||
assert "read_file" in tool_names
|
||||
|
||||
# search_files was called twice
|
||||
sf = next(t for t in tools if t["tool"] == "search_files")
|
||||
assert sf["count"] == 2
|
||||
|
||||
def test_overview_pricing_sets_are_lists(self, db):
|
||||
"""models_with/without_pricing should be JSON-serializable lists."""
|
||||
import json as _json
|
||||
db.create_session(session_id="s1", source="cli", model="gpt-4o")
|
||||
db.create_session(session_id="s2", source="cli", model="my-custom")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
overview = report["overview"]
|
||||
|
||||
assert isinstance(overview["models_with_pricing"], list)
|
||||
assert isinstance(overview["models_without_pricing"], list)
|
||||
# Should be JSON-serializable
|
||||
_json.dumps(report["overview"]) # would raise if sets present
|
||||
|
||||
def test_mixed_commercial_and_custom_models(self, db):
|
||||
"""Mix of commercial and custom models: only commercial ones get costs."""
|
||||
db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4-20250514")
|
||||
db.update_token_counts(
|
||||
"s1",
|
||||
input_tokens=10000,
|
||||
output_tokens=5000,
|
||||
billing_provider="anthropic",
|
||||
)
|
||||
db.create_session(session_id="s2", source="cli", model="my-local-llama")
|
||||
db.update_token_counts("s2", input_tokens=10000, output_tokens=5000)
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
|
||||
# Cost should only come from gpt-4o, not from the custom model
|
||||
overview = report["overview"]
|
||||
assert overview["estimated_cost"] > 0
|
||||
assert "claude-sonnet-4-20250514" in overview["models_with_pricing"] # list now, not set
|
||||
assert "my-local-llama" in overview["models_without_pricing"]
|
||||
|
||||
# Verify individual model entries
|
||||
claude = next(m for m in report["models"] if m["model"] == "claude-sonnet-4-20250514")
|
||||
assert claude["has_pricing"] is True
|
||||
assert claude["cost"] > 0
|
||||
|
||||
llama = next(m for m in report["models"] if m["model"] == "my-local-llama")
|
||||
assert llama["has_pricing"] is False
|
||||
assert llama["cost"] == 0.0
|
||||
|
||||
def test_single_session_streak(self, db):
|
||||
"""Single session should have streak of 0 or 1."""
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["activity"]["max_streak"] <= 1
|
||||
|
||||
def test_no_tool_calls(self, db):
|
||||
"""Sessions with no tool calls should produce empty tools list."""
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db.append_message("s1", role="user", content="hello")
|
||||
db.append_message("s1", role="assistant", content="hi there")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert report["tools"] == []
|
||||
|
||||
def test_only_one_platform(self, db):
|
||||
"""Single-platform usage should still work."""
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=30)
|
||||
assert len(report["platforms"]) == 1
|
||||
assert report["platforms"][0]["platform"] == "cli"
|
||||
|
||||
# Terminal format should NOT show platform section for single platform
|
||||
text = engine.format_terminal(report)
|
||||
# (it still shows platforms section if there's only cli and nothing else)
|
||||
# Actually the condition is > 1 platforms OR non-cli, so single cli won't show
|
||||
|
||||
def test_large_days_value(self, db):
|
||||
"""Very large days value should not crash."""
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=365)
|
||||
assert report["empty"] is False
|
||||
|
||||
def test_zero_days(self, db):
|
||||
"""Zero days should return empty (nothing is in the future)."""
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db._conn.commit()
|
||||
|
||||
engine = InsightsEngine(db)
|
||||
report = engine.generate(days=0)
|
||||
# Depending on timing, might catch the session if created <1s ago
|
||||
# Just verify it doesn't crash
|
||||
assert "empty" in report
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
from agent import jiter_preload
|
||||
|
||||
|
||||
def test_preload_jiter_native_extension_loads_sdk_parser_dependency():
|
||||
assert jiter_preload.preload_jiter_native_extension() is True
|
||||
assert "jiter.jiter" in sys.modules
|
||||
|
||||
|
||||
def test_preload_jiter_native_extension_is_best_effort(monkeypatch):
|
||||
monkeypatch.setattr(jiter_preload, "_JITER_PRELOADED", False)
|
||||
|
||||
def _raise_missing(name: str):
|
||||
assert name == "jiter.jiter"
|
||||
raise ModuleNotFoundError(name)
|
||||
|
||||
monkeypatch.setattr(importlib, "import_module", _raise_missing)
|
||||
|
||||
assert jiter_preload.preload_jiter_native_extension() is False
|
||||
assert jiter_preload._JITER_PRELOADED is False
|
||||
assert isinstance(jiter_preload._JITER_PRELOAD_ERROR, ModuleNotFoundError)
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Regression guard: don't send Anthropic ``thinking`` to Kimi's /coding endpoint.
|
||||
|
||||
Kimi's ``api.kimi.com/coding`` endpoint speaks the Anthropic Messages protocol
|
||||
but has its own thinking semantics. When ``thinking.enabled`` is present in
|
||||
the request, Kimi validates the message history and requires every prior
|
||||
assistant tool-call message to carry OpenAI-style ``reasoning_content``.
|
||||
|
||||
The Anthropic path never populates that field, and
|
||||
``convert_messages_to_anthropic`` strips Anthropic thinking blocks on
|
||||
third-party endpoints — so after one turn with tool calls the next request
|
||||
fails with HTTP 400::
|
||||
|
||||
thinking is enabled but reasoning_content is missing in assistant
|
||||
tool call message at index N
|
||||
|
||||
Kimi on the chat_completions route handles ``thinking`` via ``extra_body`` in
|
||||
``ChatCompletionsTransport`` (#13503). On the Anthropic route the right
|
||||
thing to do is drop the parameter entirely and let Kimi drive reasoning
|
||||
server-side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestKimiCodingSkipsAnthropicThinking:
|
||||
"""build_anthropic_kwargs must not inject ``thinking`` for Kimi /coding."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://api.kimi.com/coding",
|
||||
"https://api.kimi.com/coding/v1",
|
||||
"https://api.kimi.com/coding/anthropic",
|
||||
"https://api.kimi.com/coding/",
|
||||
],
|
||||
)
|
||||
def test_kimi_coding_endpoint_omits_thinking(self, base_url: str) -> None:
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="kimi-k2.5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url=base_url,
|
||||
)
|
||||
assert "thinking" not in kwargs, (
|
||||
"Anthropic thinking must not be sent to Kimi /coding — "
|
||||
"endpoint requires reasoning_content on history we don't preserve."
|
||||
)
|
||||
assert "output_config" not in kwargs
|
||||
|
||||
def test_kimi_coding_with_explicit_disabled_also_omits(self) -> None:
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="kimi-k2.5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": False},
|
||||
base_url="https://api.kimi.com/coding",
|
||||
)
|
||||
assert "thinking" not in kwargs
|
||||
|
||||
def test_non_kimi_third_party_still_gets_thinking(self) -> None:
|
||||
"""MiniMax and other third-party Anthropic endpoints must retain thinking."""
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="MiniMax-M2.7",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url="https://api.minimax.io/anthropic",
|
||||
)
|
||||
assert "thinking" in kwargs
|
||||
assert kwargs["thinking"]["type"] == "enabled"
|
||||
|
||||
def test_native_anthropic_still_gets_thinking(self) -> None:
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="claude-sonnet-4-20250514",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url=None,
|
||||
)
|
||||
assert "thinking" in kwargs
|
||||
|
||||
def test_kimi_root_endpoint_via_anthropic_transport_omits_thinking(self) -> None:
|
||||
"""Plain ``api.kimi.com`` hit via the Anthropic transport also omits thinking.
|
||||
|
||||
Auto-detection routes ``api.kimi.com/v1`` to ``chat_completions`` by
|
||||
default, but users can explicitly configure
|
||||
``api_mode: anthropic_messages`` against any Kimi host. The upstream
|
||||
validation (reasoning_content required on replayed tool-call
|
||||
messages) is the same regardless of URL path, so the thinking
|
||||
suppression must apply to every Kimi host, not just ``/coding``.
|
||||
See #17057.
|
||||
"""
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="kimi-k2.5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url="https://api.kimi.com/v1",
|
||||
)
|
||||
assert "thinking" not in kwargs
|
||||
|
||||
# ── #17057: custom / proxied Kimi-compatible endpoints ──────────
|
||||
@pytest.mark.parametrize(
|
||||
"base_url,model",
|
||||
[
|
||||
# Custom host with Kimi-family model — the reporter's case
|
||||
("http://my-kimi-proxy.internal", "kimi-2.6"),
|
||||
("https://llm.example.com/anthropic", "kimi-k2.5"),
|
||||
("https://llm.example.com/anthropic", "moonshot-v1-8k"),
|
||||
("https://llm.example.com/anthropic", "kimi_thinking"),
|
||||
("https://llm.example.com/anthropic", "moonshotai/kimi-k2.5"),
|
||||
# Official Moonshot host (previously uncovered)
|
||||
("https://api.moonshot.ai/anthropic", "moonshot-v1-32k"),
|
||||
("https://api.moonshot.cn/anthropic", "moonshot-v1-32k"),
|
||||
],
|
||||
)
|
||||
def test_kimi_family_custom_endpoint_omits_thinking(
|
||||
self, base_url: str, model: str
|
||||
) -> None:
|
||||
"""Custom / proxied Kimi endpoints must also strip Anthropic thinking."""
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url=base_url,
|
||||
)
|
||||
assert "thinking" not in kwargs, (
|
||||
f"Kimi-family endpoint ({base_url}, {model}) must not receive "
|
||||
f"Anthropic thinking — upstream validates reasoning_content on "
|
||||
f"replayed tool-call history we don't preserve."
|
||||
)
|
||||
assert "output_config" not in kwargs
|
||||
|
||||
def test_custom_endpoint_non_kimi_model_keeps_thinking(self) -> None:
|
||||
"""Custom endpoint with a non-Kimi model must keep thinking intact.
|
||||
|
||||
Guards against over-broad model-family matching — only model names
|
||||
starting with a Kimi/Moonshot prefix should trigger suppression.
|
||||
"""
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="MiniMax-M2.7",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url="https://my-llm-proxy.example.com/anthropic",
|
||||
)
|
||||
assert "thinking" in kwargs
|
||||
assert kwargs["thinking"]["type"] == "enabled"
|
||||
|
||||
def test_kimi_family_replay_preserves_unsigned_thinking(self) -> None:
|
||||
"""On a custom Kimi endpoint, unsigned reasoning_content thinking
|
||||
blocks must survive the third-party signature-stripping pass so
|
||||
the upstream's message-history validation passes.
|
||||
"""
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "planning the tool call",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "skill_view", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok"},
|
||||
]
|
||||
_, converted = convert_messages_to_anthropic(
|
||||
messages,
|
||||
base_url="http://my-kimi-proxy.internal",
|
||||
model="kimi-2.6",
|
||||
)
|
||||
# The assistant message still carries the unsigned thinking block
|
||||
# synthesised from reasoning_content (required by Kimi's history
|
||||
# validation). A plain third-party endpoint would have stripped it.
|
||||
assistant_msg = next(m for m in converted if m["role"] == "assistant")
|
||||
assistant_blocks = assistant_msg["content"]
|
||||
thinking_blocks = [
|
||||
b for b in assistant_blocks
|
||||
if isinstance(b, dict) and b.get("type") == "thinking"
|
||||
]
|
||||
assert len(thinking_blocks) == 1
|
||||
assert thinking_blocks[0]["thinking"] == "planning the tool call"
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Test that last_total_tokens is correctly set by ContextCompressor."""
|
||||
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
|
||||
def test_update_from_response_sets_total_tokens():
|
||||
"""ABC contract: last_total_tokens must be set from API response."""
|
||||
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
|
||||
|
||||
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30, "total_tokens": 130})
|
||||
assert c.last_total_tokens == 130
|
||||
|
||||
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30})
|
||||
assert c.last_total_tokens == 130
|
||||
|
||||
|
||||
def test_session_reset_clears_total_tokens():
|
||||
"""on_session_reset must zero total_tokens."""
|
||||
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
|
||||
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30, "total_tokens": 130})
|
||||
c.on_session_reset()
|
||||
assert c.last_total_tokens == 0
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Tests for local provider stream read timeout auto-detection.
|
||||
|
||||
When a local LLM provider is detected (Ollama, llama.cpp, vLLM, etc.),
|
||||
the httpx stream read timeout should be automatically increased from the
|
||||
default 60s to HERMES_API_TIMEOUT (1800s) to avoid premature connection
|
||||
kills during long prefill phases.
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent.model_metadata import is_local_endpoint
|
||||
|
||||
|
||||
class TestLocalStreamReadTimeout:
|
||||
"""Verify stream read timeout auto-detection logic."""
|
||||
|
||||
@pytest.mark.parametrize("base_url", [
|
||||
"http://localhost:11434",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://0.0.0.0:5000",
|
||||
"http://192.168.1.100:8000",
|
||||
"http://10.0.0.5:1234",
|
||||
"http://host.docker.internal:11434",
|
||||
"http://host.containers.internal:11434",
|
||||
"http://host.lima.internal:11434",
|
||||
])
|
||||
def test_local_endpoint_bumps_read_timeout(self, base_url):
|
||||
"""Local endpoint + default timeout -> bumps to base_timeout."""
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("HERMES_STREAM_READ_TIMEOUT", None)
|
||||
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
|
||||
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
|
||||
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
|
||||
_stream_read_timeout = _base_timeout
|
||||
assert _stream_read_timeout == 1800.0
|
||||
|
||||
def test_user_override_respected_for_local(self):
|
||||
"""User sets HERMES_STREAM_READ_TIMEOUT -> keep their value even for local."""
|
||||
with patch.dict(os.environ, {"HERMES_STREAM_READ_TIMEOUT": "300"}, clear=False):
|
||||
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
|
||||
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
|
||||
base_url = "http://localhost:11434"
|
||||
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
|
||||
_stream_read_timeout = _base_timeout
|
||||
assert _stream_read_timeout == 300.0
|
||||
|
||||
@pytest.mark.parametrize("base_url", [
|
||||
"https://api.openai.com",
|
||||
"https://openrouter.ai/api",
|
||||
"https://api.anthropic.com",
|
||||
])
|
||||
def test_remote_endpoint_keeps_default(self, base_url):
|
||||
"""Remote endpoint -> keep 120s default."""
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("HERMES_STREAM_READ_TIMEOUT", None)
|
||||
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
|
||||
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
|
||||
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
|
||||
_stream_read_timeout = _base_timeout
|
||||
assert _stream_read_timeout == 120.0
|
||||
|
||||
def test_empty_base_url_keeps_default(self):
|
||||
"""No base_url set -> keep 120s default."""
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("HERMES_STREAM_READ_TIMEOUT", None)
|
||||
_base_timeout = float(os.getenv("HERMES_API_TIMEOUT", 1800.0))
|
||||
_stream_read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
|
||||
base_url = ""
|
||||
if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
|
||||
_stream_read_timeout = _base_timeout
|
||||
assert _stream_read_timeout == 120.0
|
||||
|
||||
|
||||
class TestIsLocalEndpoint:
|
||||
"""Direct unit tests for is_local_endpoint."""
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"http://localhost:11434",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://0.0.0.0:5000",
|
||||
"http://[::1]:11434",
|
||||
"http://192.168.1.100:8000",
|
||||
"http://10.0.0.5:1234",
|
||||
"http://172.17.0.1:11434",
|
||||
])
|
||||
def test_classic_local_addresses(self, url):
|
||||
assert is_local_endpoint(url) is True
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"http://host.docker.internal:11434",
|
||||
"http://host.docker.internal:8080/v1",
|
||||
"http://gateway.docker.internal:11434",
|
||||
"http://host.containers.internal:11434",
|
||||
"http://host.lima.internal:11434",
|
||||
])
|
||||
def test_container_dns_names(self, url):
|
||||
assert is_local_endpoint(url) is True
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"https://api.openai.com",
|
||||
"https://openrouter.ai/api",
|
||||
"https://api.anthropic.com",
|
||||
"https://evil.docker.internal.example.com",
|
||||
])
|
||||
def test_remote_endpoints(self, url):
|
||||
assert is_local_endpoint(url) is False
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"http://100.64.0.0:11434", # lower bound of CGNAT block
|
||||
"http://100.64.0.1:11434/v1", # lower bound +1
|
||||
"http://100.77.243.5:11434", # representative Tailscale host
|
||||
"https://100.100.100.100:443", # Tailscale MagicDNS anchor
|
||||
"https://100.127.255.254:443", # upper bound -1
|
||||
"http://100.127.255.255:11434", # upper bound of CGNAT block
|
||||
])
|
||||
def test_tailscale_cgnat_is_local(self, url):
|
||||
"""Tailscale 100.64.0.0/10 should be treated as local for timeout bumps."""
|
||||
assert is_local_endpoint(url) is True
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"http://100.63.255.255:11434", # just below CGNAT block
|
||||
"http://100.128.0.1:11434", # just above CGNAT block
|
||||
"http://100.200.0.1:11434", # well outside CGNAT
|
||||
"http://99.64.0.1:11434", # first octet wrong
|
||||
])
|
||||
def test_near_but_not_cgnat_is_remote(self, url):
|
||||
"""Hosts adjacent to but outside 100.64.0.0/10 must not match."""
|
||||
assert is_local_endpoint(url) is False
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Tests for `agent.markdown_tables.realign_markdown_tables`.
|
||||
|
||||
These cover the alignment guarantee on CJK / wide-character tables and
|
||||
the conservative no-op behaviour on non-table input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from textwrap import dedent
|
||||
|
||||
from wcwidth import wcswidth
|
||||
|
||||
from agent.markdown_tables import (
|
||||
is_table_divider,
|
||||
looks_like_table_row,
|
||||
realign_markdown_tables,
|
||||
split_table_row,
|
||||
)
|
||||
|
||||
|
||||
def _column_offsets(line: str) -> list[int]:
|
||||
"""Return the display-cell index of every ``|`` in ``line``."""
|
||||
|
||||
cells: list[int] = []
|
||||
width = 0
|
||||
for ch in line:
|
||||
if ch == "|":
|
||||
cells.append(width)
|
||||
# wcswidth on a single char; clamp negatives.
|
||||
w = wcswidth(ch)
|
||||
width += w if w > 0 else 1
|
||||
return cells
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# split_table_row / is_table_divider / looks_like_table_row
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_split_strips_outer_pipes_and_trims():
|
||||
assert split_table_row("| a | b | c |") == ["a", "b", "c"]
|
||||
assert split_table_row("|配置|状态|") == ["配置", "状态"]
|
||||
assert split_table_row("a | b | c") == ["a", "b", "c"]
|
||||
|
||||
|
||||
def test_is_table_divider_handles_alignment_colons():
|
||||
assert is_table_divider("|---|---|")
|
||||
assert is_table_divider("| :--- | ---: | :---: |")
|
||||
assert not is_table_divider("| - | - |") # 1 dash is not a divider
|
||||
assert not is_table_divider("| a | b |")
|
||||
assert not is_table_divider("---") # single column, no pipes
|
||||
|
||||
|
||||
def test_looks_like_table_row():
|
||||
assert looks_like_table_row("| a | b |")
|
||||
assert looks_like_table_row("a | b | c") # no leading pipe, ≥2 pipes
|
||||
assert not looks_like_table_row("not a table")
|
||||
assert not looks_like_table_row("a | b") # one pipe, no leading pipe
|
||||
assert not looks_like_table_row("")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# realign_markdown_tables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_no_op_on_text_without_tables():
|
||||
text = "Hello world\nThis has no | pipes table.\n"
|
||||
assert realign_markdown_tables(text) == text
|
||||
|
||||
|
||||
def test_no_op_when_pipes_but_no_divider():
|
||||
text = "echo a | grep b\necho c | wc -l\n"
|
||||
assert realign_markdown_tables(text) == text
|
||||
|
||||
|
||||
def test_cjk_table_pipes_align_across_rows():
|
||||
# Model-emitted (under-padded for CJK) input.
|
||||
src = dedent(
|
||||
"""\
|
||||
| 配置 | Config | 论文 (%) | 复现 (%) | 差值 | 状态 |
|
||||
|------|--------|---------|---------|------|------|
|
||||
| Vicuna (report) | dense | 79.30 | 未完成 | - | × |
|
||||
| ChatGLM | chat | 37.60 | 37.82 | +0.22 | ✓ |
|
||||
| 通义千问 | qwen | (无) | 报错 | - | × |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
|
||||
# All rows in the rebuilt block must have pipes at identical display
|
||||
# columns — that's the alignment guarantee.
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
assert all(o == offsets[0] for o in offsets), (
|
||||
"rebuilt table rows do not share pipe column offsets:\n"
|
||||
+ "\n".join(out)
|
||||
)
|
||||
# And we expect 7 pipes per row (6 columns + outer borders).
|
||||
assert len(offsets[0]) == 7
|
||||
|
||||
|
||||
def test_emoji_with_cjk_table_aligns():
|
||||
src = dedent(
|
||||
"""\
|
||||
| 模型 | 状态 | 备注 |
|
||||
|------|------|------|
|
||||
| 千问 | ✅ | 通过 |
|
||||
| Claude | ✅ | 推理强 |
|
||||
| 文心一言 | ❌ | 报错 |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
# The emoji-with-variation-selector case (⚠️) intentionally tolerates
|
||||
# 1-cell drift; bare emoji like ✅ / ❌ have stable wcwidth and must
|
||||
# align. Use bare emoji here so the assertion is hard.
|
||||
assert all(o == offsets[0] for o in offsets), (
|
||||
"emoji+CJK rows do not share pipe column offsets:\n" + "\n".join(out)
|
||||
)
|
||||
|
||||
|
||||
def test_already_aligned_ascii_table_remains_aligned():
|
||||
src = dedent(
|
||||
"""\
|
||||
| a | b |
|
||||
|-----|-----|
|
||||
| 1 | 2 |
|
||||
| foo | bar |
|
||||
"""
|
||||
)
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
def test_passes_non_table_lines_through_around_a_table():
|
||||
src = dedent(
|
||||
"""\
|
||||
Here is a comparison:
|
||||
|
||||
| 模型 | 状态 |
|
||||
|------|------|
|
||||
| 千问 | 通过 |
|
||||
|
||||
And some prose after.
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src)
|
||||
assert out.startswith("Here is a comparison:\n")
|
||||
assert out.endswith("And some prose after.\n")
|
||||
# And the table lines are aligned.
|
||||
block = [ln for ln in out.split("\n") if "|" in ln]
|
||||
offsets = [_column_offsets(row) for row in block]
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vertical fallback for tables wider than the terminal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_overflow_falls_back_to_vertical_when_table_wider_than_terminal():
|
||||
"""A horizontal table that would exceed the available width must
|
||||
drop to vertical key-value rendering so the terminal does not
|
||||
soft-wrap mid-cell (which destroys column alignment visually)."""
|
||||
|
||||
src = dedent(
|
||||
"""\
|
||||
| Item | Description | Notes |
|
||||
|------|-------------|-------|
|
||||
| a | short | ok |
|
||||
| b | this is a much longer description that stretches the column wider than the others by a lot | fine |
|
||||
| c | tiny | - |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src, available_width=100)
|
||||
|
||||
# No horizontal pipe-bordered rows: vertical mode emits "Header: value"
|
||||
# lines and a ─ separator instead.
|
||||
assert "|" not in out
|
||||
assert "Item: a" in out
|
||||
assert "Description: short" in out
|
||||
assert "Notes: ok" in out
|
||||
# Body rows separated by ─ rule
|
||||
assert "──" in out
|
||||
|
||||
# Every emitted line fits the available width.
|
||||
for line in out.split("\n"):
|
||||
assert wcswidth(line) <= 100, f"line wider than budget: {line!r}"
|
||||
|
||||
|
||||
def test_horizontal_kept_when_table_fits():
|
||||
"""A table that fits the terminal must keep the horizontal
|
||||
pipe-bordered rendering — vertical fallback only kicks in when
|
||||
soft-wrap is unavoidable."""
|
||||
|
||||
src = dedent(
|
||||
"""\
|
||||
| Name | Age |
|
||||
|------|-----|
|
||||
| Alice | 30 |
|
||||
| Bob | 25 |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src, available_width=100)
|
||||
|
||||
# Pipe-bordered rendering survives.
|
||||
body_rows = [ln for ln in out.split("\n") if ln.strip().startswith("|")]
|
||||
assert len(body_rows) == 4
|
||||
offsets = [_column_offsets(r) for r in body_rows]
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
def test_vertical_fallback_wraps_long_cell_text_with_indent():
|
||||
src = dedent(
|
||||
"""\
|
||||
| Key | Value |
|
||||
|-----|-------|
|
||||
| x | this value is long enough that wrapping the value to fit a narrow terminal width is required even in vertical mode |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src, available_width=60)
|
||||
|
||||
lines = out.split("\n")
|
||||
assert lines[0].startswith("Key: x")
|
||||
# First "Value:" line + at least one continuation indented by 2 spaces.
|
||||
value_idx = next(i for i, l in enumerate(lines) if l.startswith("Value:"))
|
||||
assert lines[value_idx + 1].startswith(" ")
|
||||
# Every line still fits the budget.
|
||||
for line in lines:
|
||||
assert wcswidth(line) <= 60
|
||||
|
||||
|
||||
def test_overflow_falls_back_to_vertical_for_cjk_too():
|
||||
"""CJK content can also push a table over the terminal budget;
|
||||
the vertical fallback should kick in regardless of script."""
|
||||
|
||||
src = dedent(
|
||||
"""\
|
||||
| 模型 | 描述 | 备注 |
|
||||
|------|------|------|
|
||||
| 千问 | 一个相当长的描述用于把列宽撑得超过可用终端宽度从而触发竖排回退 | 通过 |
|
||||
| 文心 | 短 | × |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src, available_width=50)
|
||||
|
||||
assert "|" not in out
|
||||
assert "模型: 千问" in out
|
||||
assert "模型: 文心" in out
|
||||
for line in out.split("\n"):
|
||||
assert wcswidth(line) <= 50, f"line wider than budget: {line!r}"
|
||||
|
||||
|
||||
def test_handles_ragged_rows_by_padding_short_rows():
|
||||
src = dedent(
|
||||
"""\
|
||||
| a | b | c |
|
||||
|---|---|---|
|
||||
| 1 | 2 |
|
||||
| x | y | z |
|
||||
"""
|
||||
)
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
# Short rows must be padded out so they have the same pipe count
|
||||
# and column positions as the header.
|
||||
assert all(len(o) == len(offsets[0]) for o in offsets)
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
def test_multiple_tables_in_one_text():
|
||||
src = dedent(
|
||||
"""\
|
||||
First:
|
||||
|
||||
| 配置 | 值 |
|
||||
|------|----|
|
||||
| 通义 | 1 |
|
||||
|
||||
Second:
|
||||
|
||||
| model | n |
|
||||
|-------|---|
|
||||
| gpt | 2 |
|
||||
"""
|
||||
)
|
||||
out = realign_markdown_tables(src)
|
||||
# Each table block individually aligns.
|
||||
blocks: list[list[str]] = []
|
||||
current: list[str] = []
|
||||
for line in out.split("\n"):
|
||||
if "|" in line:
|
||||
current.append(line)
|
||||
elif current:
|
||||
blocks.append(current)
|
||||
current = []
|
||||
if current:
|
||||
blocks.append(current)
|
||||
|
||||
assert len(blocks) == 2
|
||||
for block in blocks:
|
||||
offsets = [_column_offsets(row) for row in block]
|
||||
assert all(o == offsets[0] for o in offsets), (
|
||||
f"block did not align:\n" + "\n".join(block)
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,327 @@
|
||||
"""Tests for the on_session_switch hook and session_id propagation.
|
||||
|
||||
Covers #6672: memory providers must be notified when AIAgent.session_id
|
||||
rotates mid-process (via /resume, /branch, /reset, /new, or context
|
||||
compression). Without the notification, providers that cache per-session
|
||||
state in initialize() (Hindsight, and any plugin that stores session_id
|
||||
for scoped writes) keep writing into the old session's record.
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.memory_manager import MemoryManager
|
||||
from agent.memory_provider import MemoryProvider
|
||||
|
||||
|
||||
class _RecordingProvider(MemoryProvider):
|
||||
"""Provider that records every lifecycle call for assertion."""
|
||||
|
||||
def __init__(self, name="rec"):
|
||||
self._name = name
|
||||
self.switch_calls: list[dict] = []
|
||||
self.sync_calls: list[dict] = []
|
||||
self.queue_calls: list[dict] = []
|
||||
self.initialize_calls: list[dict] = []
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool: # pragma: no cover - unused
|
||||
return True
|
||||
|
||||
def initialize(self, session_id, **kwargs):
|
||||
self.initialize_calls.append({"session_id": session_id, **kwargs})
|
||||
|
||||
def get_tool_schemas(self):
|
||||
return []
|
||||
|
||||
def sync_turn(self, user_content, assistant_content, *, session_id=""):
|
||||
self.sync_calls.append(
|
||||
{"user": user_content, "asst": assistant_content, "session_id": session_id}
|
||||
)
|
||||
|
||||
def queue_prefetch(self, query, *, session_id=""):
|
||||
self.queue_calls.append({"query": query, "session_id": session_id})
|
||||
|
||||
def on_session_switch(
|
||||
self,
|
||||
new_session_id,
|
||||
*,
|
||||
parent_session_id="",
|
||||
reset=False,
|
||||
**kwargs,
|
||||
):
|
||||
self.switch_calls.append(
|
||||
{
|
||||
"new": new_session_id,
|
||||
"parent": parent_session_id,
|
||||
"reset": reset,
|
||||
"extra": kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryProvider ABC — default on_session_switch is a no-op
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _MinimalProvider(MemoryProvider):
|
||||
"""Provider that does NOT override on_session_switch — ABC default must no-op."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "minimal"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(self, session_id, **kwargs): # pragma: no cover - unused
|
||||
pass
|
||||
|
||||
def get_tool_schemas(self):
|
||||
return []
|
||||
|
||||
|
||||
def test_abc_default_on_session_switch_is_noop():
|
||||
"""Providers that don't override the hook must not raise."""
|
||||
p = _MinimalProvider()
|
||||
# All three call styles must be accepted without raising
|
||||
p.on_session_switch("new-id")
|
||||
p.on_session_switch("new-id", parent_session_id="old-id")
|
||||
p.on_session_switch("new-id", parent_session_id="old-id", reset=True)
|
||||
p.on_session_switch("new-id", parent_session_id="old-id", reset=True, reason="new_session")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryManager.on_session_switch — fan-out
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manager_fans_out_to_all_providers():
|
||||
mm = MemoryManager()
|
||||
# Only one external provider is allowed; use the builtin slot for p1.
|
||||
p1 = _RecordingProvider(name="builtin")
|
||||
p2 = _RecordingProvider(name="hindsight")
|
||||
mm.add_provider(p1)
|
||||
mm.add_provider(p2)
|
||||
|
||||
mm.on_session_switch("new-sid", parent_session_id="old-sid", reset=False, reason="resume")
|
||||
|
||||
assert len(p1.switch_calls) == 1
|
||||
assert len(p2.switch_calls) == 1
|
||||
for call in (p1.switch_calls[0], p2.switch_calls[0]):
|
||||
assert call["new"] == "new-sid"
|
||||
assert call["parent"] == "old-sid"
|
||||
assert call["reset"] is False
|
||||
assert call["extra"] == {"reason": "resume"}
|
||||
|
||||
|
||||
def test_manager_ignores_empty_session_id():
|
||||
"""Empty string session_id must not trigger provider hooks.
|
||||
|
||||
Prevents accidental fires during shutdown when self.session_id may be
|
||||
cleared. Providers expect a meaningful id to switch TO.
|
||||
"""
|
||||
mm = MemoryManager()
|
||||
p = _RecordingProvider()
|
||||
mm.add_provider(p)
|
||||
mm.on_session_switch("")
|
||||
mm.on_session_switch(None) # type: ignore[arg-type]
|
||||
assert p.switch_calls == []
|
||||
|
||||
|
||||
def test_manager_isolates_provider_failures():
|
||||
"""A provider that raises must not block other providers."""
|
||||
|
||||
class _Broken(_RecordingProvider):
|
||||
def on_session_switch(self, *args, **kwargs): # type: ignore[override]
|
||||
raise RuntimeError("boom")
|
||||
|
||||
mm = MemoryManager()
|
||||
# MemoryManager rejects a second external provider, so pair broken
|
||||
# (builtin slot) with a good external one.
|
||||
broken = _Broken(name="builtin")
|
||||
good = _RecordingProvider(name="good")
|
||||
mm.add_provider(broken)
|
||||
mm.add_provider(good)
|
||||
|
||||
# Must not raise — exceptions in one provider are swallowed + logged
|
||||
mm.on_session_switch("new-sid", parent_session_id="old-sid")
|
||||
assert len(good.switch_calls) == 1
|
||||
assert good.switch_calls[0]["new"] == "new-sid"
|
||||
|
||||
|
||||
def test_manager_reset_flag_preserved():
|
||||
mm = MemoryManager()
|
||||
p = _RecordingProvider()
|
||||
mm.add_provider(p)
|
||||
mm.on_session_switch("new-sid", reset=True, reason="new_session")
|
||||
assert p.switch_calls[0]["reset"] is True
|
||||
assert p.switch_calls[0]["extra"] == {"reason": "new_session"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryManager.sync_all / queue_prefetch_all — session_id propagation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sync_all_propagates_session_id_to_providers():
|
||||
"""run_agent.py's sync_all call must pass session_id through to providers.
|
||||
|
||||
Without this, a provider that updates _session_id defensively in
|
||||
sync_turn (as Hindsight does at hindsight/__init__.py:1199) never
|
||||
sees the new id and keeps writing under the old one.
|
||||
"""
|
||||
mm = MemoryManager()
|
||||
p = _RecordingProvider()
|
||||
mm.add_provider(p)
|
||||
mm.sync_all("hello", "world", session_id="sess-42")
|
||||
assert p.sync_calls == [
|
||||
{"user": "hello", "asst": "world", "session_id": "sess-42"}
|
||||
]
|
||||
|
||||
|
||||
def test_queue_prefetch_all_propagates_session_id_to_providers():
|
||||
mm = MemoryManager()
|
||||
p = _RecordingProvider()
|
||||
mm.add_provider(p)
|
||||
mm.queue_prefetch_all("next query", session_id="sess-42")
|
||||
assert p.queue_calls == [{"query": "next query", "session_id": "sess-42"}]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hindsight reference implementation — state-flush semantics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_hindsight_provider():
|
||||
"""Build a bare HindsightMemoryProvider that skips network setup.
|
||||
|
||||
We instantiate without importing optional deps at class-level by
|
||||
bypassing __init__ and seeding the attributes on_session_switch
|
||||
reads/writes. This keeps the test hermetic.
|
||||
"""
|
||||
import threading
|
||||
hindsight_mod = pytest.importorskip("plugins.memory.hindsight")
|
||||
provider = object.__new__(hindsight_mod.HindsightMemoryProvider)
|
||||
provider._session_id = "old-sid"
|
||||
provider._parent_session_id = ""
|
||||
provider._document_id = "old-sid-20260101_000000_000000"
|
||||
provider._session_turns = ["turn-1", "turn-2"]
|
||||
provider._turn_counter = 2
|
||||
provider._turn_index = 2
|
||||
# Attrs read by _build_metadata / _build_retain_kwargs when the
|
||||
# buffer-flush path on session switch fires. Empty strings keep the
|
||||
# metadata minimal but well-formed.
|
||||
provider._retain_source = ""
|
||||
provider._platform = ""
|
||||
provider._user_id = ""
|
||||
provider._user_name = ""
|
||||
provider._chat_id = ""
|
||||
provider._chat_name = ""
|
||||
provider._chat_type = ""
|
||||
provider._thread_id = ""
|
||||
provider._agent_identity = ""
|
||||
provider._agent_workspace = ""
|
||||
provider._retain_tags = []
|
||||
provider._retain_context = "test-context"
|
||||
provider._retain_async = False
|
||||
provider._bank_id = "test-bank"
|
||||
# Prefetch state the switch path drains/clears.
|
||||
provider._prefetch_thread = None
|
||||
provider._prefetch_lock = threading.Lock()
|
||||
provider._prefetch_result = ""
|
||||
# Sync thread tracking (legacy alias at the writer).
|
||||
provider._sync_thread = None
|
||||
# Writer queue infra the flush-on-switch path enqueues onto. We stub
|
||||
# _ensure_writer / _register_atexit so no real thread is spawned;
|
||||
# tests exercising flush delivery live in
|
||||
# tests/plugins/memory/test_hindsight_provider.py where the full
|
||||
# writer-queue wiring is in place.
|
||||
import queue as _queue
|
||||
provider._retain_queue = _queue.Queue()
|
||||
provider._shutting_down = threading.Event()
|
||||
provider._atexit_registered = True
|
||||
provider._ensure_writer = lambda: None
|
||||
provider._register_atexit = lambda: None
|
||||
# Mode + API state used by _resolve_retain_target; stub the resolver
|
||||
# so tests don't actually probe the API. Real probe behavior is
|
||||
# exercised by tests in tests/plugins/memory/test_hindsight_provider.py.
|
||||
provider._mode = "cloud"
|
||||
provider._api_url = ""
|
||||
provider._api_key = ""
|
||||
provider._client = None
|
||||
provider._resolve_retain_target = lambda fb: (fb, None)
|
||||
# Stub the network-touching helper so any enqueued flush closure is
|
||||
# a no-op if ever drained in a unit test.
|
||||
provider._run_hindsight_operation = lambda _op: None
|
||||
return provider
|
||||
|
||||
|
||||
def test_hindsight_on_session_switch_updates_session_id_and_mints_fresh_doc():
|
||||
provider = _make_hindsight_provider()
|
||||
old_doc = provider._document_id
|
||||
|
||||
provider.on_session_switch(
|
||||
"new-sid", parent_session_id="old-sid", reset=False, reason="resume"
|
||||
)
|
||||
|
||||
assert provider._session_id == "new-sid"
|
||||
assert provider._parent_session_id == "old-sid"
|
||||
# Document id MUST be fresh — else next retain overwrites old session doc
|
||||
assert provider._document_id != old_doc
|
||||
assert provider._document_id.startswith("new-sid-")
|
||||
|
||||
|
||||
def test_hindsight_on_session_switch_clears_turn_buffers():
|
||||
"""Accumulated _session_turns must not leak into the next session.
|
||||
|
||||
Hindsight batches turns under a single _document_id. If the buffer
|
||||
isn't cleared on switch, the next retain under the new _document_id
|
||||
flushes turns that belong to the previous session.
|
||||
"""
|
||||
provider = _make_hindsight_provider()
|
||||
provider.on_session_switch("new-sid", parent_session_id="old-sid")
|
||||
assert provider._session_turns == []
|
||||
assert provider._turn_counter == 0
|
||||
assert provider._turn_index == 0
|
||||
|
||||
|
||||
def test_hindsight_on_session_switch_clears_on_reset_true():
|
||||
"""reset=True (from /new, /reset) must also flush buffers."""
|
||||
provider = _make_hindsight_provider()
|
||||
provider.on_session_switch("new-sid", reset=True, reason="new_session")
|
||||
assert provider._session_id == "new-sid"
|
||||
assert provider._session_turns == []
|
||||
assert provider._turn_counter == 0
|
||||
|
||||
|
||||
def test_hindsight_on_session_switch_ignores_empty_id():
|
||||
"""Empty new_session_id must be a no-op to avoid corrupting state."""
|
||||
provider = _make_hindsight_provider()
|
||||
before = (
|
||||
provider._session_id,
|
||||
provider._document_id,
|
||||
list(provider._session_turns),
|
||||
provider._turn_counter,
|
||||
)
|
||||
provider.on_session_switch("")
|
||||
provider.on_session_switch(None) # type: ignore[arg-type]
|
||||
after = (
|
||||
provider._session_id,
|
||||
provider._document_id,
|
||||
list(provider._session_turns),
|
||||
provider._turn_counter,
|
||||
)
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_hindsight_preserves_parent_across_empty_parent_arg():
|
||||
"""Omitting parent_session_id must NOT overwrite an existing one."""
|
||||
provider = _make_hindsight_provider()
|
||||
provider._parent_session_id = "original-parent"
|
||||
provider.on_session_switch("new-sid") # no parent passed
|
||||
assert provider._parent_session_id == "original-parent"
|
||||
@@ -0,0 +1,358 @@
|
||||
"""Tests for per-user memory scoping via user_id threading.
|
||||
|
||||
Verifies that gateway user_id flows from AIAgent -> MemoryManager -> plugins,
|
||||
so each gateway user gets their own memory bucket instead of sharing a static one.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from agent.memory_provider import MemoryProvider
|
||||
from agent.memory_manager import MemoryManager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concrete test provider that records init kwargs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class RecordingProvider(MemoryProvider):
|
||||
"""Minimal provider that records what initialize() receives."""
|
||||
|
||||
def __init__(self, name="recording"):
|
||||
self._name = name
|
||||
self._init_kwargs = {}
|
||||
self._init_session_id = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(self, session_id: str, **kwargs) -> None:
|
||||
self._init_session_id = session_id
|
||||
self._init_kwargs = dict(kwargs)
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
return ""
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
return ""
|
||||
|
||||
def sync_turn(self, user_content, assistant_content, *, session_id=""):
|
||||
pass
|
||||
|
||||
def get_tool_schemas(self):
|
||||
return []
|
||||
|
||||
def handle_tool_call(self, tool_name, args, **kwargs):
|
||||
return json.dumps({})
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryManager user_id threading tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMemoryManagerUserIdThreading:
|
||||
"""Verify user_id reaches providers via initialize_all."""
|
||||
|
||||
def test_user_id_forwarded_to_provider(self):
|
||||
mgr = MemoryManager()
|
||||
p = RecordingProvider()
|
||||
mgr.add_provider(p)
|
||||
|
||||
mgr.initialize_all(
|
||||
session_id="sess-123",
|
||||
platform="telegram",
|
||||
user_id="tg_user_42",
|
||||
)
|
||||
|
||||
assert p._init_kwargs.get("user_id") == "tg_user_42"
|
||||
assert p._init_kwargs.get("platform") == "telegram"
|
||||
assert p._init_session_id == "sess-123"
|
||||
|
||||
def test_chat_context_forwarded_to_provider(self):
|
||||
mgr = MemoryManager()
|
||||
p = RecordingProvider()
|
||||
mgr.add_provider(p)
|
||||
|
||||
mgr.initialize_all(
|
||||
session_id="sess-chat",
|
||||
platform="discord",
|
||||
user_id="discord_u_7",
|
||||
user_name="fakeusername",
|
||||
chat_id="1485316232612941897",
|
||||
chat_name="fakeassistantname-forums",
|
||||
chat_type="thread",
|
||||
thread_id="1491249007475949698",
|
||||
)
|
||||
|
||||
assert p._init_kwargs.get("user_name") == "fakeusername"
|
||||
assert p._init_kwargs.get("chat_id") == "1485316232612941897"
|
||||
assert p._init_kwargs.get("chat_name") == "fakeassistantname-forums"
|
||||
assert p._init_kwargs.get("chat_type") == "thread"
|
||||
assert p._init_kwargs.get("thread_id") == "1491249007475949698"
|
||||
|
||||
def test_no_user_id_when_cli(self):
|
||||
"""CLI sessions should not have user_id in kwargs."""
|
||||
mgr = MemoryManager()
|
||||
p = RecordingProvider()
|
||||
mgr.add_provider(p)
|
||||
|
||||
mgr.initialize_all(
|
||||
session_id="sess-456",
|
||||
platform="cli",
|
||||
)
|
||||
|
||||
assert "user_id" not in p._init_kwargs
|
||||
assert p._init_kwargs.get("platform") == "cli"
|
||||
|
||||
def test_user_id_none_not_forwarded(self):
|
||||
"""Explicit None user_id should not appear in kwargs."""
|
||||
mgr = MemoryManager()
|
||||
p = RecordingProvider()
|
||||
mgr.add_provider(p)
|
||||
|
||||
# Simulates what happens when AIAgent passes user_id=None
|
||||
# (the agent code only adds user_id to kwargs when it's truthy)
|
||||
mgr.initialize_all(
|
||||
session_id="sess-789",
|
||||
platform="discord",
|
||||
)
|
||||
|
||||
assert "user_id" not in p._init_kwargs
|
||||
|
||||
def test_multiple_providers_all_receive_user_id(self):
|
||||
mgr = MemoryManager()
|
||||
# Use one provider named "builtin" (always accepted) and one external
|
||||
p1 = RecordingProvider("builtin")
|
||||
p2 = RecordingProvider("external")
|
||||
mgr.add_provider(p1)
|
||||
mgr.add_provider(p2)
|
||||
|
||||
mgr.initialize_all(
|
||||
session_id="sess-multi",
|
||||
platform="slack",
|
||||
user_id="slack_U12345",
|
||||
)
|
||||
|
||||
assert p1._init_kwargs.get("user_id") == "slack_U12345"
|
||||
assert p1._init_kwargs.get("platform") == "slack"
|
||||
assert p2._init_kwargs.get("user_id") == "slack_U12345"
|
||||
assert p2._init_kwargs.get("platform") == "slack"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mem0 provider user_id tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMem0UserIdScoping:
|
||||
"""Verify Mem0 plugin uses gateway user_id when provided."""
|
||||
|
||||
def test_gateway_user_id_overrides_default(self):
|
||||
"""When user_id is passed via kwargs, it should override the config default."""
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
|
||||
provider = Mem0MemoryProvider()
|
||||
# Mock _load_config to return a config with default user_id
|
||||
with patch("plugins.memory.mem0._load_config", return_value={
|
||||
"api_key": "test-key",
|
||||
"user_id": "hermes-user",
|
||||
"agent_id": "hermes",
|
||||
"rerank": True,
|
||||
}):
|
||||
provider.initialize(session_id="test-sess", user_id="tg_user_99")
|
||||
|
||||
assert provider._user_id == "tg_user_99"
|
||||
|
||||
def test_no_user_id_falls_back_to_config(self):
|
||||
"""Without user_id in kwargs, should use config default."""
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
|
||||
provider = Mem0MemoryProvider()
|
||||
with patch("plugins.memory.mem0._load_config", return_value={
|
||||
"api_key": "test-key",
|
||||
"user_id": "custom-default",
|
||||
"agent_id": "hermes",
|
||||
"rerank": True,
|
||||
}):
|
||||
provider.initialize(session_id="test-sess")
|
||||
|
||||
assert provider._user_id == "custom-default"
|
||||
|
||||
def test_no_user_id_no_config_uses_hermes_user(self):
|
||||
"""Without user_id or config override, should default to 'hermes-user'."""
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
|
||||
provider = Mem0MemoryProvider()
|
||||
with patch("plugins.memory.mem0._load_config", return_value={
|
||||
"api_key": "test-key",
|
||||
"agent_id": "hermes",
|
||||
"rerank": True,
|
||||
}):
|
||||
provider.initialize(session_id="test-sess")
|
||||
|
||||
assert provider._user_id == "hermes-user"
|
||||
|
||||
def test_different_users_get_different_ids(self):
|
||||
"""Two providers initialized with different user_ids should be scoped differently."""
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
|
||||
p1 = Mem0MemoryProvider()
|
||||
p2 = Mem0MemoryProvider()
|
||||
|
||||
with patch("plugins.memory.mem0._load_config", return_value={
|
||||
"api_key": "test-key",
|
||||
"user_id": "hermes-user",
|
||||
"agent_id": "hermes",
|
||||
"rerank": True,
|
||||
}):
|
||||
p1.initialize(session_id="sess-1", user_id="alice_123")
|
||||
p2.initialize(session_id="sess-2", user_id="bob_456")
|
||||
|
||||
assert p1._user_id == "alice_123"
|
||||
assert p2._user_id == "bob_456"
|
||||
assert p1._user_id != p2._user_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Honcho provider user_id tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHonchoUserIdScoping:
|
||||
"""Verify Honcho plugin keeps runtime user scoping separate from config peer_name."""
|
||||
|
||||
def test_gateway_user_id_is_passed_as_runtime_peer(self):
|
||||
"""Gateway user_id should scope Honcho sessions without mutating config peer_name."""
|
||||
from plugins.memory.honcho import HonchoMemoryProvider
|
||||
|
||||
provider = HonchoMemoryProvider()
|
||||
|
||||
mock_cfg = MagicMock()
|
||||
mock_cfg.enabled = True
|
||||
mock_cfg.api_key = "test-key"
|
||||
mock_cfg.base_url = None
|
||||
mock_cfg.peer_name = "static-user"
|
||||
mock_cfg.recall_mode = "context"
|
||||
mock_cfg.context_tokens = None
|
||||
mock_cfg.raw = {}
|
||||
mock_cfg.dialectic_depth = 1
|
||||
mock_cfg.dialectic_depth_levels = None
|
||||
mock_cfg.init_on_session_start = False
|
||||
mock_cfg.ai_peer = "hermes"
|
||||
mock_cfg.resolve_session_name.return_value = "test-sess"
|
||||
mock_cfg.session_strategy = "shared"
|
||||
|
||||
with patch(
|
||||
"plugins.memory.honcho.client.HonchoClientConfig.from_global_config",
|
||||
return_value=mock_cfg,
|
||||
), patch(
|
||||
"plugins.memory.honcho.client.get_honcho_client",
|
||||
return_value=MagicMock(),
|
||||
), patch(
|
||||
"plugins.memory.honcho.session.HonchoSessionManager",
|
||||
) as mock_manager_cls:
|
||||
mock_manager = MagicMock()
|
||||
mock_manager.get_or_create.return_value = MagicMock(messages=[])
|
||||
mock_manager_cls.return_value = mock_manager
|
||||
provider.initialize(
|
||||
session_id="test-sess",
|
||||
user_id="discord_user_789",
|
||||
platform="discord",
|
||||
)
|
||||
|
||||
assert mock_cfg.peer_name == "static-user"
|
||||
assert mock_manager_cls.call_args.kwargs["runtime_user_peer_name"] == "discord_user_789"
|
||||
|
||||
def test_session_manager_prefers_runtime_user_id_over_config_peer_name(self):
|
||||
"""Session manager should isolate gateway users even when config peer_name is static."""
|
||||
from plugins.memory.honcho.session import HonchoSessionManager
|
||||
|
||||
mock_cfg = MagicMock()
|
||||
mock_cfg.peer_name = "static-user"
|
||||
mock_cfg.ai_peer = "hermes"
|
||||
mock_cfg.write_frequency = "sync"
|
||||
mock_cfg.dialectic_reasoning_level = "low"
|
||||
mock_cfg.dialectic_dynamic = True
|
||||
mock_cfg.dialectic_max_chars = 600
|
||||
mock_cfg.observation_mode = "directional"
|
||||
mock_cfg.user_observe_me = True
|
||||
mock_cfg.user_observe_others = True
|
||||
mock_cfg.ai_observe_me = True
|
||||
mock_cfg.ai_observe_others = True
|
||||
|
||||
manager = HonchoSessionManager(
|
||||
honcho=MagicMock(),
|
||||
config=mock_cfg,
|
||||
runtime_user_peer_name="discord_user_789",
|
||||
)
|
||||
|
||||
with patch.object(manager, "_get_or_create_peer", return_value=MagicMock()), patch.object(
|
||||
manager,
|
||||
"_get_or_create_honcho_session",
|
||||
return_value=(MagicMock(), []),
|
||||
):
|
||||
session = manager.get_or_create("discord:channel-1")
|
||||
|
||||
assert session.user_peer_id == "discord_user_789"
|
||||
|
||||
def test_no_user_id_preserves_config_peer_name(self):
|
||||
"""Without user_id, the config peer_name should be preserved."""
|
||||
from plugins.memory.honcho import HonchoMemoryProvider
|
||||
|
||||
provider = HonchoMemoryProvider()
|
||||
|
||||
mock_cfg = MagicMock()
|
||||
mock_cfg.enabled = True
|
||||
mock_cfg.api_key = "test-key"
|
||||
mock_cfg.base_url = None
|
||||
mock_cfg.peer_name = "my-custom-peer"
|
||||
mock_cfg.recall_mode = "tools"
|
||||
|
||||
with patch(
|
||||
"plugins.memory.honcho.client.HonchoClientConfig.from_global_config",
|
||||
return_value=mock_cfg,
|
||||
):
|
||||
provider.initialize(
|
||||
session_id="test-sess",
|
||||
platform="cli",
|
||||
)
|
||||
|
||||
# peer_name should not have been overridden
|
||||
assert mock_cfg.peer_name == "my-custom-peer"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AIAgent user_id propagation test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAIAgentUserIdPropagation:
|
||||
"""Verify AIAgent stores user_id and passes it to memory init kwargs."""
|
||||
|
||||
def test_user_id_stored_on_agent(self):
|
||||
"""AIAgent should store user_id as instance attribute."""
|
||||
with patch.dict(os.environ, {"HERMES_HOME": "/tmp/test_hermes"}):
|
||||
from run_agent import AIAgent
|
||||
agent = object.__new__(AIAgent)
|
||||
# Manually set the attribute as __init__ does
|
||||
agent._user_id = "test_user_42"
|
||||
assert agent._user_id == "test_user_42"
|
||||
|
||||
def test_user_id_none_by_default(self):
|
||||
"""AIAgent should have None user_id when not provided (CLI mode)."""
|
||||
with patch.dict(os.environ, {"HERMES_HOME": "/tmp/test_hermes"}):
|
||||
from run_agent import AIAgent
|
||||
agent = object.__new__(AIAgent)
|
||||
agent._user_id = None
|
||||
assert agent._user_id is None
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Tests for MiniMax auxiliary client URL normalization.
|
||||
|
||||
MiniMax and MiniMax-CN set inference_base_url to the /anthropic path.
|
||||
The auxiliary client uses the OpenAI SDK, which needs /v1 instead.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
|
||||
from agent.auxiliary_client import _to_openai_base_url
|
||||
|
||||
|
||||
class TestToOpenaiBaseUrl:
|
||||
def test_minimax_global_anthropic_suffix_replaced(self):
|
||||
assert _to_openai_base_url("https://api.minimax.io/anthropic") == "https://api.minimax.io/v1"
|
||||
|
||||
def test_minimax_cn_anthropic_suffix_replaced(self):
|
||||
assert _to_openai_base_url("https://api.minimaxi.com/anthropic") == "https://api.minimaxi.com/v1"
|
||||
|
||||
def test_trailing_slash_stripped_before_replace(self):
|
||||
assert _to_openai_base_url("https://api.minimax.io/anthropic/") == "https://api.minimax.io/v1"
|
||||
|
||||
def test_v1_url_unchanged(self):
|
||||
assert _to_openai_base_url("https://api.openai.com/v1") == "https://api.openai.com/v1"
|
||||
|
||||
def test_openrouter_url_unchanged(self):
|
||||
assert _to_openai_base_url("https://openrouter.ai/api/v1") == "https://openrouter.ai/api/v1"
|
||||
|
||||
def test_anthropic_domain_unchanged(self):
|
||||
"""api.anthropic.com doesn't end with /anthropic — should be untouched."""
|
||||
assert _to_openai_base_url("https://api.anthropic.com") == "https://api.anthropic.com"
|
||||
|
||||
def test_anthropic_in_subpath_unchanged(self):
|
||||
assert _to_openai_base_url("https://example.com/anthropic/extra") == "https://example.com/anthropic/extra"
|
||||
|
||||
def test_empty_string(self):
|
||||
assert _to_openai_base_url("") == ""
|
||||
|
||||
def test_none(self):
|
||||
assert _to_openai_base_url(None) == ""
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Tests for MiniMax provider hardening — context lengths, thinking, catalog, beta headers, transport."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
class TestMinimaxContextLengths:
|
||||
"""Verify context length entries match official docs (204,800 for all models).
|
||||
|
||||
Source: https://platform.minimax.io/docs/api-reference/text-anthropic-api
|
||||
"""
|
||||
|
||||
def test_minimax_prefix_has_correct_context(self):
|
||||
from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS
|
||||
assert DEFAULT_CONTEXT_LENGTHS["minimax"] == 204_800
|
||||
|
||||
def test_minimax_models_resolve_via_prefix(self):
|
||||
from agent.model_metadata import get_model_context_length
|
||||
# All MiniMax models should resolve to 204,800 via the "minimax" prefix
|
||||
for model in ("MiniMax-M2.7", "MiniMax-M2.5", "MiniMax-M2.1", "MiniMax-M2"):
|
||||
ctx = get_model_context_length(model, "")
|
||||
assert ctx == 204_800, f"{model} expected 204800, got {ctx}"
|
||||
|
||||
|
||||
|
||||
class TestMinimaxThinkingSupport:
|
||||
"""Verify that MiniMax gets manual thinking (not adaptive).
|
||||
|
||||
MiniMax's Anthropic-compat endpoint officially supports the thinking
|
||||
parameter (https://platform.minimax.io/docs/api-reference/text-anthropic-api).
|
||||
It should get manual thinking (type=enabled + budget_tokens), NOT adaptive
|
||||
thinking (which is Claude 4.6-only).
|
||||
"""
|
||||
|
||||
def test_minimax_m27_gets_manual_thinking(self):
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="MiniMax-M2.7",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
)
|
||||
assert "thinking" in kwargs
|
||||
assert kwargs["thinking"]["type"] == "enabled"
|
||||
assert "budget_tokens" in kwargs["thinking"]
|
||||
# MiniMax should NOT get adaptive thinking or output_config
|
||||
assert "output_config" not in kwargs
|
||||
|
||||
def test_minimax_m25_gets_manual_thinking(self):
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="MiniMax-M2.5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
)
|
||||
assert "thinking" in kwargs
|
||||
assert kwargs["thinking"]["type"] == "enabled"
|
||||
|
||||
def test_thinking_still_works_for_claude(self):
|
||||
from agent.anthropic_adapter import build_anthropic_kwargs
|
||||
kwargs = build_anthropic_kwargs(
|
||||
model="claude-sonnet-4-20250514",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
max_tokens=4096,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
)
|
||||
assert "thinking" in kwargs
|
||||
|
||||
|
||||
class TestMinimaxAuxModel:
|
||||
"""Verify auxiliary model is standard (not highspeed) — now reads from profiles."""
|
||||
|
||||
def test_minimax_aux_is_standard(self):
|
||||
from agent.auxiliary_client import _get_aux_model_for_provider
|
||||
assert _get_aux_model_for_provider("minimax") == "MiniMax-M2.7"
|
||||
assert _get_aux_model_for_provider("minimax-cn") == "MiniMax-M2.7"
|
||||
|
||||
def test_minimax_aux_not_highspeed(self):
|
||||
from agent.auxiliary_client import _get_aux_model_for_provider
|
||||
assert "highspeed" not in _get_aux_model_for_provider("minimax")
|
||||
assert "highspeed" not in _get_aux_model_for_provider("minimax-cn")
|
||||
|
||||
|
||||
class TestMinimaxBetaHeaders:
|
||||
"""MiniMax Anthropic-compat endpoints reject fine-grained-tool-streaming beta.
|
||||
|
||||
Verify that build_anthropic_client omits the tool-streaming beta for MiniMax
|
||||
(both global and China domains) while keeping it for native Anthropic and
|
||||
other third-party endpoints. Covers the fix for #6510 / #6555.
|
||||
"""
|
||||
|
||||
_TOOL_BETA = "fine-grained-tool-streaming-2025-05-14"
|
||||
_THINKING_BETA = "interleaved-thinking-2025-05-14"
|
||||
|
||||
# -- helper ----------------------------------------------------------
|
||||
|
||||
def _build_and_get_betas(self, api_key, base_url=None):
|
||||
"""Build client, return the anthropic-beta header string."""
|
||||
from agent.anthropic_adapter import build_anthropic_client
|
||||
with patch("agent.anthropic_adapter._anthropic_sdk") as mock_sdk:
|
||||
build_anthropic_client(api_key, base_url=base_url)
|
||||
kwargs = mock_sdk.Anthropic.call_args[1]
|
||||
headers = kwargs.get("default_headers", {})
|
||||
return headers.get("anthropic-beta", "")
|
||||
|
||||
# -- MiniMax global --------------------------------------------------
|
||||
|
||||
def test_minimax_global_omits_tool_streaming(self):
|
||||
betas = self._build_and_get_betas(
|
||||
"mm-key-123", base_url="https://api.minimax.io/anthropic"
|
||||
)
|
||||
assert self._TOOL_BETA not in betas
|
||||
assert self._THINKING_BETA in betas
|
||||
|
||||
def test_minimax_global_trailing_slash(self):
|
||||
betas = self._build_and_get_betas(
|
||||
"mm-key-123", base_url="https://api.minimax.io/anthropic/"
|
||||
)
|
||||
assert self._TOOL_BETA not in betas
|
||||
|
||||
# -- MiniMax China ---------------------------------------------------
|
||||
|
||||
def test_minimax_cn_omits_tool_streaming(self):
|
||||
betas = self._build_and_get_betas(
|
||||
"mm-cn-key-456", base_url="https://api.minimaxi.com/anthropic"
|
||||
)
|
||||
assert self._TOOL_BETA not in betas
|
||||
assert self._THINKING_BETA in betas
|
||||
|
||||
def test_minimax_cn_trailing_slash(self):
|
||||
betas = self._build_and_get_betas(
|
||||
"mm-cn-key-456", base_url="https://api.minimaxi.com/anthropic/"
|
||||
)
|
||||
assert self._TOOL_BETA not in betas
|
||||
|
||||
# -- Non-MiniMax keeps full betas ------------------------------------
|
||||
|
||||
def test_native_anthropic_keeps_tool_streaming(self):
|
||||
betas = self._build_and_get_betas("sk-ant-api03-real-key-here")
|
||||
assert self._TOOL_BETA in betas
|
||||
assert self._THINKING_BETA in betas
|
||||
|
||||
def test_third_party_proxy_keeps_tool_streaming(self):
|
||||
betas = self._build_and_get_betas(
|
||||
"custom-key", base_url="https://my-proxy.example.com/anthropic"
|
||||
)
|
||||
assert self._TOOL_BETA in betas
|
||||
|
||||
def test_custom_base_url_keeps_tool_streaming(self):
|
||||
betas = self._build_and_get_betas(
|
||||
"custom-key", base_url="https://custom.api.com"
|
||||
)
|
||||
assert self._TOOL_BETA in betas
|
||||
|
||||
# -- _common_betas_for_base_url unit tests ---------------------------
|
||||
|
||||
def test_common_betas_none_url(self):
|
||||
from agent.anthropic_adapter import _common_betas_for_base_url, _COMMON_BETAS
|
||||
assert _common_betas_for_base_url(None) == _COMMON_BETAS
|
||||
|
||||
def test_common_betas_empty_url(self):
|
||||
from agent.anthropic_adapter import _common_betas_for_base_url, _COMMON_BETAS
|
||||
assert _common_betas_for_base_url("") == _COMMON_BETAS
|
||||
|
||||
def test_common_betas_minimax_url(self):
|
||||
from agent.anthropic_adapter import _common_betas_for_base_url, _TOOL_STREAMING_BETA
|
||||
betas = _common_betas_for_base_url("https://api.minimax.io/anthropic")
|
||||
assert _TOOL_STREAMING_BETA not in betas
|
||||
assert len(betas) > 0 # still has other betas
|
||||
|
||||
def test_common_betas_minimax_cn_url(self):
|
||||
from agent.anthropic_adapter import _common_betas_for_base_url, _TOOL_STREAMING_BETA
|
||||
betas = _common_betas_for_base_url("https://api.minimaxi.com/anthropic")
|
||||
assert _TOOL_STREAMING_BETA not in betas
|
||||
|
||||
def test_common_betas_regular_url(self):
|
||||
from agent.anthropic_adapter import _common_betas_for_base_url, _COMMON_BETAS
|
||||
assert _common_betas_for_base_url("https://api.anthropic.com") == _COMMON_BETAS
|
||||
|
||||
|
||||
class TestMinimaxApiMode:
|
||||
"""Verify determine_api_mode returns anthropic_messages for MiniMax providers.
|
||||
|
||||
The MiniMax /anthropic endpoint speaks Anthropic Messages wire format,
|
||||
not OpenAI chat completions. The overlay transport must reflect this
|
||||
so that code paths calling determine_api_mode() without a base_url
|
||||
(e.g. /model switch) get the correct api_mode.
|
||||
"""
|
||||
|
||||
def test_minimax_returns_anthropic_messages(self):
|
||||
from hermes_cli.providers import determine_api_mode
|
||||
assert determine_api_mode("minimax") == "anthropic_messages"
|
||||
|
||||
def test_minimax_cn_returns_anthropic_messages(self):
|
||||
from hermes_cli.providers import determine_api_mode
|
||||
assert determine_api_mode("minimax-cn") == "anthropic_messages"
|
||||
|
||||
def test_minimax_with_url_also_works(self):
|
||||
from hermes_cli.providers import determine_api_mode
|
||||
# Even with explicit base_url, provider lookup takes priority
|
||||
assert determine_api_mode("minimax", "https://api.minimax.io/anthropic") == "anthropic_messages"
|
||||
|
||||
def test_anthropic_still_returns_anthropic_messages(self):
|
||||
from hermes_cli.providers import determine_api_mode
|
||||
assert determine_api_mode("anthropic") == "anthropic_messages"
|
||||
|
||||
def test_openai_returns_chat_completions(self):
|
||||
from hermes_cli.providers import determine_api_mode
|
||||
# Sanity check: standard providers are unaffected
|
||||
result = determine_api_mode("deepseek")
|
||||
assert result == "chat_completions"
|
||||
|
||||
|
||||
class TestMinimaxMaxOutput:
|
||||
"""Verify _get_anthropic_max_output returns correct limits for MiniMax models.
|
||||
|
||||
MiniMax max output is 131,072 tokens (source: OpenClaw model definitions,
|
||||
cross-referenced with MiniMax API behavior).
|
||||
"""
|
||||
|
||||
def test_minimax_m27_output_limit(self):
|
||||
from agent.anthropic_adapter import _get_anthropic_max_output
|
||||
assert _get_anthropic_max_output("MiniMax-M2.7") == 131_072
|
||||
|
||||
def test_minimax_m25_output_limit(self):
|
||||
from agent.anthropic_adapter import _get_anthropic_max_output
|
||||
assert _get_anthropic_max_output("MiniMax-M2.5") == 131_072
|
||||
|
||||
def test_minimax_m2_output_limit(self):
|
||||
from agent.anthropic_adapter import _get_anthropic_max_output
|
||||
assert _get_anthropic_max_output("MiniMax-M2") == 131_072
|
||||
|
||||
def test_claude_output_unaffected(self):
|
||||
from agent.anthropic_adapter import _get_anthropic_max_output
|
||||
# Sanity: Claude limits are not broken by the MiniMax entry
|
||||
assert _get_anthropic_max_output("claude-sonnet-4-6") == 64_000
|
||||
|
||||
|
||||
class TestMinimaxPreserveDots:
|
||||
"""Verify that MiniMax model names preserve dots through the Anthropic adapter.
|
||||
|
||||
MiniMax model IDs like 'MiniMax-M2.7' must NOT have dots converted to
|
||||
hyphens — the endpoint expects the exact name with dots.
|
||||
"""
|
||||
|
||||
def test_minimax_provider_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="minimax", base_url="")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_minimax_cn_provider_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="minimax-cn", base_url="")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_minimax_url_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="custom", base_url="https://api.minimax.io/anthropic")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_minimax_cn_url_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="custom", base_url="https://api.minimaxi.com/anthropic")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_anthropic_does_not_preserve_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="anthropic", base_url="https://api.anthropic.com")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is False
|
||||
|
||||
def test_opencode_zen_provider_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="opencode-zen", base_url="")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_opencode_zen_url_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="custom", base_url="https://opencode.ai/zen/v1")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_zai_provider_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="zai", base_url="")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_bigmodel_cn_url_preserves_dots(self):
|
||||
from types import SimpleNamespace
|
||||
agent = SimpleNamespace(provider="custom", base_url="https://open.bigmodel.cn/api/paas/v4")
|
||||
from run_agent import AIAgent
|
||||
assert AIAgent._anthropic_preserve_dots(agent) is True
|
||||
|
||||
def test_normalize_preserves_m25_free_dot(self):
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name("minimax-m2.5-free", preserve_dots=True) == "minimax-m2.5-free"
|
||||
|
||||
def test_normalize_preserves_m27_dot(self):
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name("MiniMax-M2.7", preserve_dots=True) == "MiniMax-M2.7"
|
||||
|
||||
def test_normalize_preserves_non_anthropic_dots_without_preserve(self):
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
# Non-Anthropic model families use dots as canonical version separators;
|
||||
# only Claude/Anthropic names are hyphen-normalized by default.
|
||||
assert normalize_model_name("MiniMax-M2.7", preserve_dots=False) == "MiniMax-M2.7"
|
||||
|
||||
def test_normalize_still_converts_claude_dots_without_preserve(self):
|
||||
from agent.anthropic_adapter import normalize_model_name
|
||||
assert normalize_model_name("claude-opus-4.6", preserve_dots=False) == "claude-opus-4-6"
|
||||
|
||||
|
||||
class TestMinimaxSwitchModelCredentialGuard:
|
||||
"""Verify switch_model() does not leak Anthropic credentials to MiniMax.
|
||||
|
||||
The __init__ path correctly guards against this (line 761), but switch_model()
|
||||
must mirror that guard. Without it, /model switch to minimax with no explicit
|
||||
api_key would fall back to resolve_anthropic_token() and send Anthropic creds
|
||||
to the MiniMax endpoint.
|
||||
"""
|
||||
|
||||
def test_switch_to_minimax_does_not_resolve_anthropic_token(self):
|
||||
"""switch_model() should NOT call resolve_anthropic_token() for MiniMax."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
with patch("run_agent.AIAgent.__init__", return_value=None):
|
||||
from run_agent import AIAgent
|
||||
agent = AIAgent.__new__(AIAgent)
|
||||
agent.provider = "anthropic"
|
||||
agent.model = "claude-sonnet-4"
|
||||
agent.api_key = "sk-ant-fake"
|
||||
agent.base_url = "https://api.anthropic.com"
|
||||
agent.api_mode = "anthropic_messages"
|
||||
agent._anthropic_base_url = "https://api.anthropic.com"
|
||||
agent._anthropic_api_key = "sk-ant-fake"
|
||||
agent._is_anthropic_oauth = False
|
||||
agent._client_kwargs = {}
|
||||
agent.client = None
|
||||
agent._anthropic_client = MagicMock()
|
||||
agent._fallback_chain = []
|
||||
|
||||
with patch("agent.anthropic_adapter.build_anthropic_client") as mock_build, \
|
||||
patch("agent.anthropic_adapter.resolve_anthropic_token", return_value="sk-ant-leaked") as mock_resolve, \
|
||||
patch("agent.anthropic_adapter._is_oauth_token", return_value=False):
|
||||
|
||||
agent.switch_model(
|
||||
new_model="MiniMax-M2.7",
|
||||
new_provider="minimax",
|
||||
api_mode="anthropic_messages",
|
||||
api_key="mm-key-123",
|
||||
base_url="https://api.minimax.io/anthropic",
|
||||
)
|
||||
# resolve_anthropic_token should NOT be called for non-Anthropic providers
|
||||
mock_resolve.assert_not_called()
|
||||
# The key passed to build_anthropic_client should be the MiniMax key
|
||||
build_args = mock_build.call_args
|
||||
assert build_args[0][0] == "mm-key-123"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,597 @@
|
||||
"""Tests for _query_local_context_length and the local server fallback in
|
||||
get_model_context_length.
|
||||
|
||||
All tests use synthetic inputs — no filesystem or live server required.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _query_local_context_length — unit tests with mocked httpx
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestQueryLocalContextLengthOllama:
|
||||
"""_query_local_context_length with server_type == 'ollama'."""
|
||||
|
||||
def _make_resp(self, status_code, body):
|
||||
resp = MagicMock()
|
||||
resp.status_code = status_code
|
||||
resp.json.return_value = body
|
||||
return resp
|
||||
|
||||
def test_ollama_model_info_context_length(self):
|
||||
"""Reads context length from model_info dict in /api/show response."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
show_resp = self._make_resp(200, {
|
||||
"model_info": {"llama.context_length": 131072}
|
||||
})
|
||||
models_resp = self._make_resp(404, {})
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = show_resp
|
||||
client_mock.get.return_value = models_resp
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="ollama"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("omnicoder-9b", "http://localhost:11434/v1")
|
||||
|
||||
assert result == 131072
|
||||
|
||||
def test_ollama_parameters_num_ctx(self):
|
||||
"""Falls back to num_ctx in parameters string when model_info lacks context_length."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
show_resp = self._make_resp(200, {
|
||||
"model_info": {},
|
||||
"parameters": "num_ctx 32768\ntemperature 0.7\n"
|
||||
})
|
||||
models_resp = self._make_resp(404, {})
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = show_resp
|
||||
client_mock.get.return_value = models_resp
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="ollama"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("some-model", "http://localhost:11434/v1")
|
||||
|
||||
assert result == 32768
|
||||
|
||||
def test_ollama_num_ctx_wins_over_model_info(self):
|
||||
"""When both num_ctx (Modelfile) and model_info (GGUF) are present,
|
||||
num_ctx wins because it's the *runtime* context Ollama actually
|
||||
allocates KV cache for. The GGUF model_info.context_length is the
|
||||
training max — using it would let Hermes grow conversations past
|
||||
the runtime limit and Ollama would silently truncate.
|
||||
|
||||
Concrete example: hermes-brain:qwen3-14b-ctx32k is a Modelfile
|
||||
derived from qwen3:14b with `num_ctx 32768`, but the underlying
|
||||
GGUF reports `qwen3.context_length: 40960` (training max). If
|
||||
Hermes used 40960 it would let the conversation grow past 32768
|
||||
before compressing, and Ollama would truncate the prefix.
|
||||
"""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
show_resp = self._make_resp(200, {
|
||||
"model_info": {"qwen3.context_length": 40960},
|
||||
"parameters": "num_ctx 32768\ntemperature 0.6\n",
|
||||
})
|
||||
models_resp = self._make_resp(404, {})
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = show_resp
|
||||
client_mock.get.return_value = models_resp
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="ollama"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length(
|
||||
"hermes-brain:qwen3-14b-ctx32k", "http://100.77.243.5:11434/v1"
|
||||
)
|
||||
|
||||
assert result == 32768, (
|
||||
f"Expected num_ctx (32768) to win over model_info (40960), got {result}. "
|
||||
"If Hermes uses the GGUF training max, conversations will silently truncate."
|
||||
)
|
||||
|
||||
def test_ollama_show_404_falls_through(self):
|
||||
"""When /api/show returns 404, falls through to /v1/models/{model}."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
show_resp = self._make_resp(404, {})
|
||||
model_detail_resp = self._make_resp(200, {"max_model_len": 65536})
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = show_resp
|
||||
client_mock.get.return_value = model_detail_resp
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="ollama"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("some-model", "http://localhost:11434/v1")
|
||||
|
||||
assert result == 65536
|
||||
|
||||
|
||||
class TestQueryLocalContextLengthVllm:
|
||||
"""_query_local_context_length with vLLM-style /v1/models/{model} response."""
|
||||
|
||||
def _make_resp(self, status_code, body):
|
||||
resp = MagicMock()
|
||||
resp.status_code = status_code
|
||||
resp.json.return_value = body
|
||||
return resp
|
||||
|
||||
def test_vllm_max_model_len(self):
|
||||
"""Reads max_model_len from /v1/models/{model} response."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
detail_resp = self._make_resp(200, {"id": "omnicoder-9b", "max_model_len": 100000})
|
||||
list_resp = self._make_resp(404, {})
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = self._make_resp(404, {})
|
||||
client_mock.get.return_value = detail_resp
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="vllm"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("omnicoder-9b", "http://localhost:8000/v1")
|
||||
|
||||
assert result == 100000
|
||||
|
||||
def test_vllm_context_length_key(self):
|
||||
"""Reads context_length from /v1/models/{model} response."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
detail_resp = self._make_resp(200, {"id": "some-model", "context_length": 32768})
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = self._make_resp(404, {})
|
||||
client_mock.get.return_value = detail_resp
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="vllm"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("some-model", "http://localhost:8000/v1")
|
||||
|
||||
assert result == 32768
|
||||
|
||||
|
||||
class TestQueryLocalContextLengthModelsList:
|
||||
"""_query_local_context_length: falls back to /v1/models list."""
|
||||
|
||||
def _make_resp(self, status_code, body):
|
||||
resp = MagicMock()
|
||||
resp.status_code = status_code
|
||||
resp.json.return_value = body
|
||||
return resp
|
||||
|
||||
def test_models_list_max_model_len(self):
|
||||
"""Finds context length for model in /v1/models list."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
detail_resp = self._make_resp(404, {})
|
||||
list_resp = self._make_resp(200, {
|
||||
"data": [
|
||||
{"id": "other-model", "max_model_len": 4096},
|
||||
{"id": "omnicoder-9b", "max_model_len": 131072},
|
||||
]
|
||||
})
|
||||
|
||||
call_count = [0]
|
||||
def side_effect(url, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
return detail_resp # /v1/models/omnicoder-9b
|
||||
return list_resp # /v1/models
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = self._make_resp(404, {})
|
||||
client_mock.get.side_effect = side_effect
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value=None), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("omnicoder-9b", "http://localhost:1234")
|
||||
|
||||
assert result == 131072
|
||||
|
||||
def test_models_list_model_not_found_returns_none(self):
|
||||
"""Returns None when model is not in the /v1/models list."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
detail_resp = self._make_resp(404, {})
|
||||
list_resp = self._make_resp(200, {
|
||||
"data": [{"id": "other-model", "max_model_len": 4096}]
|
||||
})
|
||||
|
||||
call_count = [0]
|
||||
def side_effect(url, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
return detail_resp
|
||||
return list_resp
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = self._make_resp(404, {})
|
||||
client_mock.get.side_effect = side_effect
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value=None), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("omnicoder-9b", "http://localhost:1234")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestQueryLocalContextLengthLmStudio:
|
||||
"""_query_local_context_length with LM Studio native /api/v1/models response."""
|
||||
|
||||
def _make_resp(self, status_code, body):
|
||||
resp = MagicMock()
|
||||
resp.status_code = status_code
|
||||
resp.json.return_value = body
|
||||
return resp
|
||||
|
||||
def _make_client(self, native_resp, detail_resp, list_resp):
|
||||
"""Build a mock httpx.Client with sequenced GET responses."""
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.return_value = self._make_resp(404, {})
|
||||
|
||||
responses = [native_resp, detail_resp, list_resp]
|
||||
call_idx = [0]
|
||||
|
||||
def get_side_effect(url, **kwargs):
|
||||
idx = call_idx[0]
|
||||
call_idx[0] += 1
|
||||
if idx < len(responses):
|
||||
return responses[idx]
|
||||
return self._make_resp(404, {})
|
||||
|
||||
client_mock.get.side_effect = get_side_effect
|
||||
return client_mock
|
||||
|
||||
def test_lmstudio_exact_key_match(self):
|
||||
"""Resolves loaded ctx when key matches exactly."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
native_resp = self._make_resp(200, {
|
||||
"models": [
|
||||
{"key": "nvidia/nvidia-nemotron-super-49b-v1",
|
||||
"id": "nvidia/nvidia-nemotron-super-49b-v1",
|
||||
"max_context_length": 1_048_576,
|
||||
"loaded_instances": [{"config": {"context_length": 131072}}]},
|
||||
]
|
||||
})
|
||||
client_mock = self._make_client(
|
||||
native_resp,
|
||||
self._make_resp(404, {}),
|
||||
self._make_resp(404, {}),
|
||||
)
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="lm-studio"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length(
|
||||
"nvidia/nvidia-nemotron-super-49b-v1", "http://192.168.1.22:1234/v1"
|
||||
)
|
||||
|
||||
assert result == 131072
|
||||
|
||||
def test_lmstudio_slug_only_matches_key_with_publisher_prefix(self):
|
||||
"""Fuzzy match: bare model slug matches key that includes publisher prefix.
|
||||
|
||||
When the user configures the model as "local:nvidia-nemotron-super-49b-v1"
|
||||
(slug only, no publisher), but LM Studio's native API stores it as
|
||||
"nvidia/nvidia-nemotron-super-49b-v1", the lookup must still succeed.
|
||||
"""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
native_resp = self._make_resp(200, {
|
||||
"models": [
|
||||
{"key": "nvidia/nvidia-nemotron-super-49b-v1",
|
||||
"id": "nvidia/nvidia-nemotron-super-49b-v1",
|
||||
"max_context_length": 1_048_576,
|
||||
"loaded_instances": [{"config": {"context_length": 131072}}]},
|
||||
]
|
||||
})
|
||||
client_mock = self._make_client(
|
||||
native_resp,
|
||||
self._make_resp(404, {}),
|
||||
self._make_resp(404, {}),
|
||||
)
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="lm-studio"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
# Model passed in is just the slug after stripping "local:" prefix
|
||||
result = _query_local_context_length(
|
||||
"nvidia-nemotron-super-49b-v1", "http://192.168.1.22:1234/v1"
|
||||
)
|
||||
|
||||
assert result == 131072
|
||||
|
||||
def test_lmstudio_v1_models_list_slug_fuzzy_match(self):
|
||||
"""Fuzzy match also works for /v1/models list when exact match fails.
|
||||
|
||||
LM Studio's OpenAI-compat /v1/models returns id like
|
||||
"nvidia/nvidia-nemotron-super-49b-v1" — must match bare slug.
|
||||
"""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
# native /api/v1/models: no match
|
||||
native_resp = self._make_resp(404, {})
|
||||
# /v1/models/{model}: no match
|
||||
detail_resp = self._make_resp(404, {})
|
||||
# /v1/models list: model found with publisher prefix, includes context_length
|
||||
list_resp = self._make_resp(200, {
|
||||
"data": [
|
||||
{"id": "nvidia/nvidia-nemotron-super-49b-v1", "context_length": 131072},
|
||||
]
|
||||
})
|
||||
client_mock = self._make_client(native_resp, detail_resp, list_resp)
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="lm-studio"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length(
|
||||
"nvidia-nemotron-super-49b-v1", "http://192.168.1.22:1234/v1"
|
||||
)
|
||||
|
||||
assert result == 131072
|
||||
|
||||
def test_lmstudio_loaded_instances_context_length(self):
|
||||
"""Reads active context_length from loaded_instances when max_context_length absent."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
native_resp = self._make_resp(200, {
|
||||
"models": [
|
||||
{
|
||||
"key": "nvidia/nvidia-nemotron-super-49b-v1",
|
||||
"id": "nvidia/nvidia-nemotron-super-49b-v1",
|
||||
"loaded_instances": [
|
||||
{"config": {"context_length": 65536}},
|
||||
],
|
||||
},
|
||||
]
|
||||
})
|
||||
client_mock = self._make_client(
|
||||
native_resp,
|
||||
self._make_resp(404, {}),
|
||||
self._make_resp(404, {}),
|
||||
)
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="lm-studio"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length(
|
||||
"nvidia-nemotron-super-49b-v1", "http://192.168.1.22:1234/v1"
|
||||
)
|
||||
|
||||
assert result == 65536
|
||||
|
||||
def test_lmstudio_loaded_instance_beats_max_context_length(self):
|
||||
"""loaded_instances context_length takes priority over max_context_length.
|
||||
|
||||
LM Studio may show max_context_length=1_048_576 (theoretical model max)
|
||||
while the actual loaded context is 122_651 (runtime setting). The loaded
|
||||
value is the real constraint and must be preferred.
|
||||
"""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
native_resp = self._make_resp(200, {
|
||||
"models": [
|
||||
{
|
||||
"key": "nvidia/nvidia-nemotron-3-nano-4b",
|
||||
"id": "nvidia/nvidia-nemotron-3-nano-4b",
|
||||
"max_context_length": 1_048_576,
|
||||
"loaded_instances": [
|
||||
{"config": {"context_length": 122_651}},
|
||||
],
|
||||
},
|
||||
]
|
||||
})
|
||||
client_mock = self._make_client(
|
||||
native_resp,
|
||||
self._make_resp(404, {}),
|
||||
self._make_resp(404, {}),
|
||||
)
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="lm-studio"), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length(
|
||||
"nvidia-nemotron-3-nano-4b", "http://192.168.1.22:1234/v1"
|
||||
)
|
||||
|
||||
assert result == 122_651, (
|
||||
f"Expected loaded instance context (122651) but got {result}. "
|
||||
"max_context_length (1048576) must not win over loaded_instances."
|
||||
)
|
||||
|
||||
|
||||
class TestDetectLocalServerTypeAuth:
|
||||
def test_passes_bearer_token_to_probe_requests(self):
|
||||
from agent.model_metadata import detect_local_server_type
|
||||
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.get.return_value = resp
|
||||
|
||||
with patch("httpx.Client", return_value=client_mock) as mock_client:
|
||||
result = detect_local_server_type("http://localhost:1234/v1", api_key="lm-token")
|
||||
|
||||
assert result == "lm-studio"
|
||||
assert mock_client.call_args.kwargs["headers"] == {
|
||||
"Authorization": "Bearer lm-token"
|
||||
}
|
||||
|
||||
|
||||
class TestFetchEndpointModelMetadataLmStudio:
|
||||
"""fetch_endpoint_model_metadata should use LM Studio's native models endpoint."""
|
||||
|
||||
def _make_resp(self, body):
|
||||
resp = MagicMock()
|
||||
resp.raise_for_status.return_value = None
|
||||
resp.json.return_value = body
|
||||
return resp
|
||||
|
||||
def test_uses_native_models_endpoint_only(self):
|
||||
from agent.model_metadata import fetch_endpoint_model_metadata
|
||||
|
||||
native_resp = self._make_resp(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"key": "lmstudio-community/Qwen3.5-27B-GGUF/Qwen3.5-27B-Q8_0.gguf",
|
||||
"id": "lmstudio-community/Qwen3.5-27B-GGUF/Qwen3.5-27B-Q8_0.gguf",
|
||||
"max_context_length": 1_048_576,
|
||||
"loaded_instances": [
|
||||
{"config": {"context_length": 131072}}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value="lm-studio"), \
|
||||
patch("agent.model_metadata.requests.get", return_value=native_resp) as mock_get:
|
||||
result = fetch_endpoint_model_metadata(
|
||||
"http://localhost:1234/v1",
|
||||
api_key="lm-token",
|
||||
force_refresh=True,
|
||||
)
|
||||
|
||||
assert mock_get.call_count == 1
|
||||
assert mock_get.call_args[0][0] == "http://localhost:1234/api/v1/models"
|
||||
assert mock_get.call_args.kwargs["headers"] == {
|
||||
"Authorization": "Bearer lm-token"
|
||||
}
|
||||
assert result["lmstudio-community/Qwen3.5-27B-GGUF/Qwen3.5-27B-Q8_0.gguf"]["context_length"] == 131072
|
||||
assert result["Qwen3.5-27B-GGUF/Qwen3.5-27B-Q8_0.gguf"]["context_length"] == 131072
|
||||
|
||||
|
||||
class TestQueryLocalContextLengthNetworkError:
|
||||
"""_query_local_context_length handles network failures gracefully."""
|
||||
|
||||
def test_connection_error_returns_none(self):
|
||||
"""Returns None when the server is unreachable."""
|
||||
from agent.model_metadata import _query_local_context_length
|
||||
|
||||
client_mock = MagicMock()
|
||||
client_mock.__enter__ = lambda s: client_mock
|
||||
client_mock.__exit__ = MagicMock(return_value=False)
|
||||
client_mock.post.side_effect = Exception("Connection refused")
|
||||
client_mock.get.side_effect = Exception("Connection refused")
|
||||
|
||||
with patch("agent.model_metadata.detect_local_server_type", return_value=None), \
|
||||
patch("httpx.Client", return_value=client_mock):
|
||||
result = _query_local_context_length("omnicoder-9b", "http://localhost:11434/v1")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_model_context_length — integration-style tests with mocked helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGetModelContextLengthLocalFallback:
|
||||
"""get_model_context_length uses local server query before falling back to 2M."""
|
||||
|
||||
def test_local_endpoint_unknown_model_queries_server(self):
|
||||
"""Unknown model on local endpoint gets ctx from server, not 2M default."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
|
||||
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.is_local_endpoint", return_value=True), \
|
||||
patch("agent.model_metadata._query_local_context_length", return_value=131072), \
|
||||
patch("agent.model_metadata.save_context_length") as mock_save:
|
||||
result = get_model_context_length("omnicoder-9b", "http://localhost:11434/v1")
|
||||
|
||||
assert result == 131072
|
||||
|
||||
def test_local_endpoint_unknown_model_result_is_cached(self):
|
||||
"""Context length returned from local server is persisted to cache."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
|
||||
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.is_local_endpoint", return_value=True), \
|
||||
patch("agent.model_metadata._query_local_context_length", return_value=131072), \
|
||||
patch("agent.model_metadata.save_context_length") as mock_save:
|
||||
get_model_context_length("omnicoder-9b", "http://localhost:11434/v1")
|
||||
|
||||
mock_save.assert_called_once_with("omnicoder-9b", "http://localhost:11434/v1", 131072)
|
||||
|
||||
def test_local_endpoint_server_returns_none_falls_back_to_2m(self):
|
||||
"""When local server returns None, still falls back to 2M probe tier."""
|
||||
from agent.model_metadata import get_model_context_length, CONTEXT_PROBE_TIERS
|
||||
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
|
||||
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.is_local_endpoint", return_value=True), \
|
||||
patch("agent.model_metadata._query_local_context_length", return_value=None):
|
||||
result = get_model_context_length("omnicoder-9b", "http://localhost:11434/v1")
|
||||
|
||||
assert result == CONTEXT_PROBE_TIERS[0]
|
||||
|
||||
def test_non_local_endpoint_does_not_query_local_server(self):
|
||||
"""For non-local endpoints, _query_local_context_length is not called."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
|
||||
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.is_local_endpoint", return_value=False), \
|
||||
patch("agent.model_metadata._query_local_context_length") as mock_query:
|
||||
result = get_model_context_length(
|
||||
"unknown-model", "https://some-cloud-api.example.com/v1"
|
||||
)
|
||||
|
||||
mock_query.assert_not_called()
|
||||
|
||||
def test_cached_result_skips_local_query(self):
|
||||
"""Cached context length is returned without querying the local server."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=65536), \
|
||||
patch("agent.model_metadata._query_local_context_length") as mock_query:
|
||||
result = get_model_context_length("omnicoder-9b", "http://localhost:11434/v1")
|
||||
|
||||
assert result == 65536
|
||||
mock_query.assert_not_called()
|
||||
|
||||
def test_no_base_url_does_not_query_local_server(self):
|
||||
"""When base_url is empty, local server is not queried."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
|
||||
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
|
||||
patch("agent.model_metadata._query_local_context_length") as mock_query:
|
||||
result = get_model_context_length("unknown-xyz-model", "")
|
||||
|
||||
mock_query.assert_not_called()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Tests for _resolve_requests_verify() env var precedence.
|
||||
|
||||
Verifies that custom provider `/models` fetches honour the three supported
|
||||
CA bundle env vars (HERMES_CA_BUNDLE, REQUESTS_CA_BUNDLE, SSL_CERT_FILE)
|
||||
in the documented priority order, and that non-existent paths are
|
||||
skipped gracefully rather than breaking the request.
|
||||
|
||||
No filesystem or network I/O required — we use tmp_path to create real
|
||||
CA bundle stand-in files and monkeypatch env vars.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.model_metadata import _resolve_requests_verify
|
||||
|
||||
|
||||
_CA_ENV_VARS = ("HERMES_CA_BUNDLE", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_env(monkeypatch):
|
||||
"""Clear all three SSL env vars so each test starts from a known state."""
|
||||
for var in _CA_ENV_VARS:
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
return monkeypatch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bundle_file(tmp_path: Path) -> str:
|
||||
"""Create a placeholder CA bundle file and return its absolute path."""
|
||||
path = tmp_path / "ca.pem"
|
||||
path.write_text("-----BEGIN CERTIFICATE-----\nstub\n-----END CERTIFICATE-----\n")
|
||||
return str(path)
|
||||
|
||||
|
||||
class TestResolveRequestsVerify:
|
||||
def test_no_env_returns_true(self, clean_env):
|
||||
assert _resolve_requests_verify() is True
|
||||
|
||||
def test_hermes_ca_bundle_returns_path(self, clean_env, bundle_file):
|
||||
clean_env.setenv("HERMES_CA_BUNDLE", bundle_file)
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
|
||||
def test_requests_ca_bundle_returns_path(self, clean_env, bundle_file):
|
||||
clean_env.setenv("REQUESTS_CA_BUNDLE", bundle_file)
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
|
||||
def test_ssl_cert_file_returns_path(self, clean_env, bundle_file):
|
||||
clean_env.setenv("SSL_CERT_FILE", bundle_file)
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
|
||||
def test_priority_hermes_over_requests(self, clean_env, tmp_path, bundle_file):
|
||||
other = tmp_path / "other.pem"
|
||||
other.write_text("stub")
|
||||
clean_env.setenv("HERMES_CA_BUNDLE", bundle_file)
|
||||
clean_env.setenv("REQUESTS_CA_BUNDLE", str(other))
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
|
||||
def test_priority_requests_over_ssl_cert_file(self, clean_env, tmp_path, bundle_file):
|
||||
other = tmp_path / "other.pem"
|
||||
other.write_text("stub")
|
||||
clean_env.setenv("REQUESTS_CA_BUNDLE", bundle_file)
|
||||
clean_env.setenv("SSL_CERT_FILE", str(other))
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
|
||||
def test_nonexistent_path_falls_through(self, clean_env, tmp_path, bundle_file):
|
||||
missing = tmp_path / "does_not_exist.pem"
|
||||
clean_env.setenv("HERMES_CA_BUNDLE", str(missing))
|
||||
clean_env.setenv("REQUESTS_CA_BUNDLE", bundle_file)
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
|
||||
def test_all_nonexistent_returns_true(self, clean_env, tmp_path):
|
||||
missing1 = tmp_path / "a.pem"
|
||||
missing2 = tmp_path / "b.pem"
|
||||
missing3 = tmp_path / "c.pem"
|
||||
clean_env.setenv("HERMES_CA_BUNDLE", str(missing1))
|
||||
clean_env.setenv("REQUESTS_CA_BUNDLE", str(missing2))
|
||||
clean_env.setenv("SSL_CERT_FILE", str(missing3))
|
||||
assert _resolve_requests_verify() is True
|
||||
|
||||
def test_empty_string_env_var_ignored(self, clean_env, bundle_file):
|
||||
clean_env.setenv("HERMES_CA_BUNDLE", "")
|
||||
clean_env.setenv("REQUESTS_CA_BUNDLE", bundle_file)
|
||||
assert _resolve_requests_verify() == bundle_file
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Tests for agent.models_dev — models.dev registry integration."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from agent.models_dev import (
|
||||
PROVIDER_TO_MODELS_DEV,
|
||||
_extract_context,
|
||||
fetch_models_dev,
|
||||
get_model_capabilities,
|
||||
lookup_models_dev_context,
|
||||
)
|
||||
|
||||
|
||||
SAMPLE_REGISTRY = {
|
||||
"anthropic": {
|
||||
"id": "anthropic",
|
||||
"name": "Anthropic",
|
||||
"models": {
|
||||
"claude-opus-4-6": {
|
||||
"id": "claude-opus-4-6",
|
||||
"limit": {"context": 1000000, "output": 128000},
|
||||
},
|
||||
"claude-sonnet-4-6": {
|
||||
"id": "claude-sonnet-4-6",
|
||||
"limit": {"context": 1000000, "output": 64000},
|
||||
},
|
||||
"claude-sonnet-4-0": {
|
||||
"id": "claude-sonnet-4-0",
|
||||
"limit": {"context": 200000, "output": 64000},
|
||||
},
|
||||
},
|
||||
},
|
||||
"github-copilot": {
|
||||
"id": "github-copilot",
|
||||
"name": "GitHub Copilot",
|
||||
"models": {
|
||||
"claude-opus-4.6": {
|
||||
"id": "claude-opus-4.6",
|
||||
"limit": {"context": 128000, "output": 32000},
|
||||
},
|
||||
},
|
||||
},
|
||||
"xai": {
|
||||
"id": "xai",
|
||||
"name": "xAI",
|
||||
"models": {
|
||||
"grok-build-0.1": {
|
||||
"id": "grok-build-0.1",
|
||||
"limit": {"context": 256000, "output": 64000},
|
||||
},
|
||||
},
|
||||
},
|
||||
"kilo": {
|
||||
"id": "kilo",
|
||||
"name": "Kilo Gateway",
|
||||
"models": {
|
||||
"anthropic/claude-sonnet-4.6": {
|
||||
"id": "anthropic/claude-sonnet-4.6",
|
||||
"limit": {"context": 1000000, "output": 128000},
|
||||
},
|
||||
},
|
||||
},
|
||||
"deepseek": {
|
||||
"id": "deepseek",
|
||||
"name": "DeepSeek",
|
||||
"models": {
|
||||
"deepseek-chat": {
|
||||
"id": "deepseek-chat",
|
||||
"limit": {"context": 128000, "output": 8192},
|
||||
},
|
||||
},
|
||||
},
|
||||
"audio-only": {
|
||||
"id": "audio-only",
|
||||
"models": {
|
||||
"tts-model": {
|
||||
"id": "tts-model",
|
||||
"limit": {"context": 0, "output": 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestProviderMapping:
|
||||
def test_xai_oauth_uses_xai_catalog(self):
|
||||
assert PROVIDER_TO_MODELS_DEV["xai"] == "xai"
|
||||
assert PROVIDER_TO_MODELS_DEV["xai-oauth"] == "xai"
|
||||
|
||||
def test_unmapped_provider_not_in_dict(self):
|
||||
assert "nous" not in PROVIDER_TO_MODELS_DEV
|
||||
|
||||
def test_openai_codex_mapped_to_openai(self):
|
||||
assert PROVIDER_TO_MODELS_DEV["openai"] == "openai"
|
||||
assert PROVIDER_TO_MODELS_DEV["openai-codex"] == "openai"
|
||||
|
||||
|
||||
class TestExtractContext:
|
||||
def test_valid_entry(self):
|
||||
assert _extract_context({"limit": {"context": 128000}}) == 128000
|
||||
|
||||
def test_zero_context_returns_none(self):
|
||||
assert _extract_context({"limit": {"context": 0}}) is None
|
||||
|
||||
def test_missing_limit_returns_none(self):
|
||||
assert _extract_context({"id": "test"}) is None
|
||||
|
||||
def test_missing_context_returns_none(self):
|
||||
assert _extract_context({"limit": {"output": 8192}}) is None
|
||||
|
||||
def test_non_dict_returns_none(self):
|
||||
assert _extract_context("not a dict") is None
|
||||
|
||||
def test_float_context_coerced_to_int(self):
|
||||
assert _extract_context({"limit": {"context": 131072.0}}) == 131072
|
||||
|
||||
|
||||
class TestLookupModelsDevContext:
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_exact_match(self, mock_fetch):
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
assert lookup_models_dev_context("anthropic", "claude-opus-4-6") == 1000000
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_case_insensitive_match(self, mock_fetch):
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
assert lookup_models_dev_context("anthropic", "Claude-Opus-4-6") == 1000000
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_provider_not_mapped(self, mock_fetch):
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
assert lookup_models_dev_context("nous", "some-model") is None
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_model_not_found(self, mock_fetch):
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
assert lookup_models_dev_context("anthropic", "nonexistent-model") is None
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_provider_aware_context(self, mock_fetch):
|
||||
"""Same model, different context per provider."""
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
# Anthropic direct: 1M
|
||||
assert lookup_models_dev_context("anthropic", "claude-opus-4-6") == 1000000
|
||||
# GitHub Copilot: only 128K for same model
|
||||
assert lookup_models_dev_context("copilot", "claude-opus-4.6") == 128000
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_xai_oauth_resolves_xai_context(self, mock_fetch):
|
||||
"""xAI OAuth is an auth path, not a separate model catalog."""
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
assert lookup_models_dev_context("xai-oauth", "grok-build-0.1") == 256000
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_zero_context_filtered(self, mock_fetch):
|
||||
mock_fetch.return_value = SAMPLE_REGISTRY
|
||||
# audio-only is not a mapped provider, but test the filtering directly
|
||||
data = SAMPLE_REGISTRY["audio-only"]["models"]["tts-model"]
|
||||
assert _extract_context(data) is None
|
||||
|
||||
@patch("agent.models_dev.fetch_models_dev")
|
||||
def test_empty_registry(self, mock_fetch):
|
||||
mock_fetch.return_value = {}
|
||||
assert lookup_models_dev_context("anthropic", "claude-opus-4-6") is None
|
||||
|
||||
|
||||
class TestFetchModelsDev:
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_fetch_success(self, mock_get):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json.return_value = SAMPLE_REGISTRY
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
# Clear caches
|
||||
import agent.models_dev as md
|
||||
md._models_dev_cache = {}
|
||||
md._models_dev_cache_time = 0
|
||||
|
||||
with patch.object(md, "_save_disk_cache"):
|
||||
result = fetch_models_dev(force_refresh=True)
|
||||
|
||||
assert "anthropic" in result
|
||||
assert len(result) == len(SAMPLE_REGISTRY)
|
||||
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_fetch_failure_returns_stale_cache(self, mock_get):
|
||||
mock_get.side_effect = Exception("network error")
|
||||
|
||||
import agent.models_dev as md
|
||||
md._models_dev_cache = SAMPLE_REGISTRY
|
||||
md._models_dev_cache_time = 0 # expired
|
||||
|
||||
with patch.object(md, "_load_disk_cache", return_value=SAMPLE_REGISTRY):
|
||||
result = fetch_models_dev(force_refresh=True)
|
||||
|
||||
assert "anthropic" in result
|
||||
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_in_memory_cache_used(self, mock_get):
|
||||
import agent.models_dev as md
|
||||
import time
|
||||
md._models_dev_cache = SAMPLE_REGISTRY
|
||||
md._models_dev_cache_time = time.time() # fresh
|
||||
|
||||
result = fetch_models_dev()
|
||||
mock_get.assert_not_called()
|
||||
assert result == SAMPLE_REGISTRY
|
||||
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_fresh_disk_cache_skips_network(self, mock_get):
|
||||
"""When in-mem cache is empty but disk cache exists and is fresh by
|
||||
mtime (< TTL), fetch_models_dev returns disk data without ever
|
||||
making the network call.
|
||||
|
||||
This is the cold-start fast path: every fresh process previously
|
||||
paid ~500 ms re-fetching a registry that was already on disk
|
||||
from an earlier run.
|
||||
"""
|
||||
import agent.models_dev as md
|
||||
# Empty in-mem cache so stage 1 doesn't short-circuit.
|
||||
md._models_dev_cache = {}
|
||||
md._models_dev_cache_time = 0
|
||||
|
||||
with patch.object(md, "_disk_cache_age_seconds", return_value=60.0), \
|
||||
patch.object(md, "_load_disk_cache", return_value=SAMPLE_REGISTRY):
|
||||
result = fetch_models_dev()
|
||||
|
||||
# The whole point: no network call.
|
||||
mock_get.assert_not_called()
|
||||
assert "anthropic" in result
|
||||
# In-mem cache populated so subsequent calls within the same
|
||||
# process stay on stage 1.
|
||||
assert md._models_dev_cache == SAMPLE_REGISTRY
|
||||
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_stale_disk_cache_falls_through_to_network(self, mock_get):
|
||||
"""When the disk cache is OLDER than TTL, we must hit the network
|
||||
(and only fall back to the stale disk data if network fails)."""
|
||||
import agent.models_dev as md
|
||||
md._models_dev_cache = {}
|
||||
md._models_dev_cache_time = 0
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json.return_value = SAMPLE_REGISTRY
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
# Disk cache exists but is older than the TTL — must NOT short-circuit.
|
||||
with patch.object(md, "_disk_cache_age_seconds",
|
||||
return_value=md._MODELS_DEV_CACHE_TTL + 60), \
|
||||
patch.object(md, "_load_disk_cache", return_value=SAMPLE_REGISTRY), \
|
||||
patch.object(md, "_save_disk_cache"):
|
||||
result = fetch_models_dev()
|
||||
|
||||
mock_get.assert_called_once()
|
||||
assert "anthropic" in result
|
||||
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_force_refresh_skips_disk_cache(self, mock_get):
|
||||
"""force_refresh=True bypasses BOTH the in-mem cache AND the
|
||||
disk-cache fast path. Used by ``hermes config refresh`` and
|
||||
anywhere else the user explicitly asked for fresh data.
|
||||
"""
|
||||
import agent.models_dev as md
|
||||
md._models_dev_cache = {}
|
||||
md._models_dev_cache_time = 0
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json.return_value = SAMPLE_REGISTRY
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
# Disk cache is fresh, but force_refresh must override it.
|
||||
with patch.object(md, "_disk_cache_age_seconds", return_value=60.0), \
|
||||
patch.object(md, "_load_disk_cache", return_value=SAMPLE_REGISTRY), \
|
||||
patch.object(md, "_save_disk_cache"):
|
||||
result = fetch_models_dev(force_refresh=True)
|
||||
|
||||
mock_get.assert_called_once()
|
||||
assert "anthropic" in result
|
||||
|
||||
@patch("agent.models_dev.requests.get")
|
||||
def test_missing_disk_cache_falls_through_to_network(self, mock_get):
|
||||
"""If the disk cache file doesn't exist (first-ever run, or it
|
||||
was deleted), fall through cleanly to network."""
|
||||
import agent.models_dev as md
|
||||
md._models_dev_cache = {}
|
||||
md._models_dev_cache_time = 0
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json.return_value = SAMPLE_REGISTRY
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
with patch.object(md, "_disk_cache_age_seconds", return_value=None), \
|
||||
patch.object(md, "_save_disk_cache"):
|
||||
result = fetch_models_dev()
|
||||
|
||||
mock_get.assert_called_once()
|
||||
assert "anthropic" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_model_capabilities — vision via modalities.input
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
CAPS_REGISTRY = {
|
||||
"google": {
|
||||
"id": "google",
|
||||
"models": {
|
||||
"gemma-4-31b-it": {
|
||||
"id": "gemma-4-31b-it",
|
||||
"attachment": False,
|
||||
"tool_call": True,
|
||||
"modalities": {"input": ["text", "image"]},
|
||||
"limit": {"context": 128000, "output": 8192},
|
||||
},
|
||||
"gemma-3-1b": {
|
||||
"id": "gemma-3-1b",
|
||||
"tool_call": True,
|
||||
"limit": {"context": 32000, "output": 8192},
|
||||
},
|
||||
"text-only-with-stale-attachment": {
|
||||
"id": "text-only-with-stale-attachment",
|
||||
"attachment": True,
|
||||
"tool_call": True,
|
||||
"modalities": {"input": ["text"]},
|
||||
"limit": {"context": 128000, "output": 8192},
|
||||
},
|
||||
},
|
||||
},
|
||||
"anthropic": {
|
||||
"id": "anthropic",
|
||||
"models": {
|
||||
"claude-sonnet-4": {
|
||||
"id": "claude-sonnet-4",
|
||||
"attachment": True,
|
||||
"tool_call": True,
|
||||
"limit": {"context": 200000, "output": 64000},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestGetModelCapabilities:
|
||||
"""Tests for get_model_capabilities vision detection."""
|
||||
|
||||
def test_vision_from_attachment_flag(self):
|
||||
"""Models with attachment=True and no modalities should report supports_vision=True."""
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY):
|
||||
caps = get_model_capabilities("anthropic", "claude-sonnet-4")
|
||||
assert caps is not None
|
||||
assert caps.supports_vision is True
|
||||
|
||||
def test_vision_from_modalities_input_image(self):
|
||||
"""Models with 'image' in modalities.input but attachment=False should
|
||||
still report supports_vision=True (the core fix in this PR)."""
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY):
|
||||
caps = get_model_capabilities("google", "gemma-4-31b-it")
|
||||
assert caps is not None
|
||||
assert caps.supports_vision is True
|
||||
|
||||
def test_text_only_modalities_override_stale_attachment_flag(self):
|
||||
"""Text-only modalities must win over stale attachment=True metadata."""
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY):
|
||||
caps = get_model_capabilities("google", "text-only-with-stale-attachment")
|
||||
assert caps is not None
|
||||
assert caps.supports_vision is False
|
||||
|
||||
def test_no_vision_without_attachment_or_modalities(self):
|
||||
"""Models with neither attachment nor image modality should be non-vision."""
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY):
|
||||
caps = get_model_capabilities("google", "gemma-3-1b")
|
||||
assert caps is not None
|
||||
assert caps.supports_vision is False
|
||||
|
||||
def test_modalities_non_dict_handled(self):
|
||||
"""Non-dict modalities field should not crash."""
|
||||
registry = {
|
||||
"google": {"id": "google", "models": {
|
||||
"weird-model": {
|
||||
"id": "weird-model",
|
||||
"modalities": "text", # not a dict
|
||||
"limit": {"context": 200000, "output": 8192},
|
||||
},
|
||||
}},
|
||||
}
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=registry):
|
||||
caps = get_model_capabilities("gemini", "weird-model")
|
||||
assert caps is not None
|
||||
assert caps.supports_vision is False
|
||||
|
||||
def test_model_not_found_returns_none(self):
|
||||
"""Unknown model should return None."""
|
||||
with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY):
|
||||
caps = get_model_capabilities("anthropic", "nonexistent-model")
|
||||
assert caps is None
|
||||
@@ -0,0 +1,562 @@
|
||||
"""Tests for Moonshot/Kimi flavored-JSON-Schema sanitizer.
|
||||
|
||||
Moonshot's tool-parameter validator rejects several shapes that the rest of
|
||||
the JSON Schema ecosystem accepts:
|
||||
|
||||
1. Properties without ``type`` — Moonshot requires ``type`` on every node.
|
||||
2. ``type`` at the parent of ``anyOf`` — Moonshot requires it only inside
|
||||
``anyOf`` children.
|
||||
3. ``$ref`` with sibling keywords — Moonshot expands the ref first and then
|
||||
rejects ``description``/``type`` siblings on the same node.
|
||||
(Ported from anomalyco/opencode#24730.)
|
||||
4. Tuple-style ``items`` arrays — Moonshot requires a single item schema,
|
||||
not positional ones. (Ported from anomalyco/opencode#24730.)
|
||||
|
||||
These tests cover the repairs applied by ``agent/moonshot_schema.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.moonshot_schema import (
|
||||
is_moonshot_model,
|
||||
sanitize_moonshot_tool_parameters,
|
||||
sanitize_moonshot_tools,
|
||||
)
|
||||
|
||||
|
||||
class TestMoonshotModelDetection:
|
||||
"""is_moonshot_model() must match across aggregator prefixes."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"kimi-k2.6",
|
||||
"kimi-k2-thinking",
|
||||
"moonshotai/Kimi-K2.6",
|
||||
"moonshotai/kimi-k2.6",
|
||||
"nous/moonshotai/kimi-k2.6",
|
||||
"openrouter/moonshotai/kimi-k2-thinking",
|
||||
"MOONSHOTAI/KIMI-K2.6",
|
||||
],
|
||||
)
|
||||
def test_positive_matches(self, model):
|
||||
assert is_moonshot_model(model) is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"",
|
||||
None,
|
||||
"anthropic/claude-sonnet-4.6",
|
||||
"openai/gpt-5.4",
|
||||
"google/gemini-3-flash-preview",
|
||||
"deepseek-chat",
|
||||
],
|
||||
)
|
||||
def test_negative_matches(self, model):
|
||||
assert is_moonshot_model(model) is False
|
||||
|
||||
|
||||
class TestMissingTypeFilled:
|
||||
"""Rule 1: every property must carry a type."""
|
||||
|
||||
def test_property_without_type_gets_string(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {"query": {"description": "a bare property"}},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["query"]["type"] == "string"
|
||||
|
||||
def test_property_with_enum_infers_type_from_first_value(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {"flag": {"enum": [True, False]}},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["flag"]["type"] == "boolean"
|
||||
|
||||
def test_nested_properties_are_repaired(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filter": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"field": {"description": "no type"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["filter"]["properties"]["field"]["type"] == "string"
|
||||
|
||||
def test_array_items_without_type_get_repaired(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {"description": "tag entry"},
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["tags"]["items"]["type"] == "string"
|
||||
|
||||
def test_ref_node_is_not_given_synthetic_type(self):
|
||||
"""$ref nodes should NOT get a synthetic type — the referenced
|
||||
definition supplies it, and Moonshot would reject the conflict."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {"payload": {"$ref": "#/$defs/Payload"}},
|
||||
"$defs": {"Payload": {"type": "object", "properties": {}}},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert "type" not in out["properties"]["payload"]
|
||||
assert out["properties"]["payload"]["$ref"] == "#/$defs/Payload"
|
||||
|
||||
|
||||
class TestAnyOfParentType:
|
||||
"""Rule 2: type must not appear at the anyOf parent level.
|
||||
|
||||
When an anyOf contains a null-type branch, Moonshot rejects it.
|
||||
The sanitizer collapses the anyOf: single non-null branch is promoted,
|
||||
multiple non-null branches have null removed from the list.
|
||||
"""
|
||||
|
||||
def test_anyof_null_branch_collapsed_to_single_type(self):
|
||||
"""anyOf [string, null] → plain string (anyOf removed)."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from_format": {
|
||||
"type": "string",
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
{"type": "null"},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
from_format = out["properties"]["from_format"]
|
||||
# null branch removed, anyOf collapsed to the single non-null type
|
||||
assert "anyOf" not in from_format
|
||||
assert from_format["type"] == "string"
|
||||
|
||||
def test_anyof_multiple_non_null_preserved(self):
|
||||
"""anyOf [string, integer] (no null) → kept as-is with parent type stripped."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
{"type": "integer"},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
mode = out["properties"]["mode"]
|
||||
assert "anyOf" in mode
|
||||
assert "type" not in mode # parent type stripped
|
||||
|
||||
def test_anyof_enum_with_null_collapsed(self):
|
||||
"""anyOf [{enum: [...], type: string}, {type: null}] → enum + type only."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db_type": {
|
||||
"anyOf": [
|
||||
{"enum": ["mysql", "postgresql", ""]},
|
||||
{"type": "null"},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
db_type = out["properties"]["db_type"]
|
||||
assert "anyOf" not in db_type
|
||||
assert db_type["type"] == "string"
|
||||
assert db_type["enum"] == ["mysql", "postgresql"] # "" stripped by enum cleanup
|
||||
|
||||
|
||||
class TestRefSiblingStripping:
|
||||
"""Rule 4: ``$ref`` nodes may not carry sibling keywords on Moonshot.
|
||||
|
||||
Ported from anomalyco/opencode#24730. The real-world failure was MCP tools
|
||||
whose generated schemas put a ``description`` on a ``$ref`` property so the
|
||||
model would see the field's human-readable hint. The reference stays — the
|
||||
referenced definition still owns the description (on the target node itself)
|
||||
and still serves the model's context.
|
||||
"""
|
||||
|
||||
def test_description_sibling_stripped_from_ref(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"variantOptions": {
|
||||
"$ref": "#/$defs/VariantOptions",
|
||||
"description": "Required. The variant options for generation.",
|
||||
},
|
||||
},
|
||||
"$defs": {
|
||||
"VariantOptions": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"description": "Configuration options.",
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
# Sibling stripped.
|
||||
assert out["properties"]["variantOptions"] == {"$ref": "#/$defs/VariantOptions"}
|
||||
# The target definition's own description is preserved — we only strip
|
||||
# siblings ON the $ref node, not on the thing it points at.
|
||||
assert out["$defs"]["VariantOptions"]["description"] == "Configuration options."
|
||||
|
||||
def test_multiple_siblings_all_stripped(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"p": {
|
||||
"$ref": "#/$defs/T",
|
||||
"type": "object",
|
||||
"description": "x",
|
||||
"default": {},
|
||||
"title": "P",
|
||||
},
|
||||
},
|
||||
"$defs": {"T": {"type": "object"}},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["p"] == {"$ref": "#/$defs/T"}
|
||||
|
||||
def test_ref_without_siblings_unchanged(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {"p": {"$ref": "#/$defs/T"}},
|
||||
"$defs": {"T": {"type": "object"}},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["p"] == {"$ref": "#/$defs/T"}
|
||||
|
||||
def test_ref_inside_anyof_children(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"v": {
|
||||
"anyOf": [
|
||||
{"$ref": "#/$defs/A", "description": "variant A"},
|
||||
{"type": "null"},
|
||||
],
|
||||
},
|
||||
},
|
||||
"$defs": {"A": {"type": "object"}},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
# Main's existing Rule 2 collapses anyOf-with-null down to the
|
||||
# single non-null branch (Moonshot rejects null branches in anyOf
|
||||
# outright). That branch was originally `{"$ref": ..., "description": ...}`;
|
||||
# Rule 4 then strips the sibling, leaving exactly `{"$ref": "..."}`.
|
||||
# The test name still applies — Rule 4 ran on the $ref branch — it
|
||||
# just happens after the anyOf collapse on this input.
|
||||
assert out["properties"]["v"] == {"$ref": "#/$defs/A"}
|
||||
|
||||
|
||||
class TestTupleItems:
|
||||
"""Rule 5: tuple-style ``items`` arrays collapse to a single schema.
|
||||
|
||||
Ported from anomalyco/opencode#24730. Moonshot's schema engine requires
|
||||
``items`` to be ONE schema object applied to every array element; tuple-
|
||||
style positional item schemas are rejected. We collapse to the first
|
||||
element's schema (which is the "closest" interpretation of positional →
|
||||
single) and drop the rest.
|
||||
"""
|
||||
|
||||
def test_tuple_items_collapsed_to_first(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"renderedSize": {
|
||||
"type": "array",
|
||||
"items": [{"type": "number"}, {"type": "number"}],
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["renderedSize"]["items"] == {"type": "number"}
|
||||
# Sibling constraints are preserved — only the tuple shape is repaired.
|
||||
assert out["properties"]["renderedSize"]["minItems"] == 2
|
||||
|
||||
def test_empty_tuple_items_becomes_empty_schema(self):
|
||||
# Empty tuple collapses to ``{}``; the generic repair then fills a
|
||||
# synthetic ``type`` because Moonshot requires ``type`` on every
|
||||
# schema node. Either ``{}`` or ``{"type": "string"}`` is a valid
|
||||
# final shape for Moonshot — both accept any string element — but we
|
||||
# always go through ``_fill_missing_type`` so the result is fully
|
||||
# well-formed without needing the consumer to patch it later.
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"things": {"type": "array", "items": []},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
items = out["properties"]["things"]["items"]
|
||||
# Must be a dict and must carry a ``type`` (the whole point of Rule 1).
|
||||
assert isinstance(items, dict)
|
||||
assert items.get("type")
|
||||
|
||||
def test_tuple_items_first_element_is_repaired(self):
|
||||
# The first element itself has a missing type — it should be filled.
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pair": {
|
||||
"type": "array",
|
||||
"items": [{"description": "first"}, {"description": "second"}],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
# Repaired to a single schema with a synthetic type.
|
||||
assert out["properties"]["pair"]["items"] == {
|
||||
"description": "first",
|
||||
"type": "string",
|
||||
}
|
||||
|
||||
def test_single_schema_items_unchanged(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tags": {"type": "array", "items": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["tags"]["items"] == {"type": "string"}
|
||||
|
||||
|
||||
class TestTopLevelGuarantees:
|
||||
"""The returned top-level schema is always a well-formed object."""
|
||||
|
||||
def test_non_dict_input_returns_empty_object(self):
|
||||
assert sanitize_moonshot_tool_parameters(None) == {"type": "object", "properties": {}}
|
||||
assert sanitize_moonshot_tool_parameters("garbage") == {"type": "object", "properties": {}}
|
||||
assert sanitize_moonshot_tool_parameters([]) == {"type": "object", "properties": {}}
|
||||
|
||||
def test_non_object_top_level_coerced(self):
|
||||
params = {"type": "string"}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["type"] == "object"
|
||||
assert "properties" in out
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {"q": {"description": "no type"}},
|
||||
}
|
||||
snapshot = {
|
||||
"type": params["type"],
|
||||
"properties": {"q": dict(params["properties"]["q"])},
|
||||
}
|
||||
sanitize_moonshot_tool_parameters(params)
|
||||
assert params["type"] == snapshot["type"]
|
||||
assert "type" not in params["properties"]["q"]
|
||||
|
||||
|
||||
class TestToolListSanitizer:
|
||||
"""sanitize_moonshot_tools() walks an OpenAI-format tool list."""
|
||||
|
||||
def test_applies_per_tool(self):
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"description": "Search",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"q": {"description": "query"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "noop",
|
||||
"description": "Does nothing",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
},
|
||||
]
|
||||
out = sanitize_moonshot_tools(tools)
|
||||
assert out[0]["function"]["parameters"]["properties"]["q"]["type"] == "string"
|
||||
# Second tool already clean — should be structurally equivalent
|
||||
assert out[1]["function"]["parameters"] == {"type": "object", "properties": {}}
|
||||
|
||||
def test_empty_list_is_passthrough(self):
|
||||
assert sanitize_moonshot_tools([]) == []
|
||||
assert sanitize_moonshot_tools(None) is None
|
||||
|
||||
def test_skips_malformed_entries(self):
|
||||
"""Entries without a function dict are passed through untouched."""
|
||||
tools = [{"type": "function"}, {"not": "a tool"}]
|
||||
out = sanitize_moonshot_tools(tools)
|
||||
assert out == tools
|
||||
|
||||
|
||||
class TestRealWorldMCPShape:
|
||||
"""End-to-end: a realistic MCP-style schema that used to 400 on Moonshot."""
|
||||
|
||||
def test_combined_rewrites(self):
|
||||
# Shape: missing type on a property, anyOf with parent type + null, array
|
||||
# items without type — all in one tool.
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"description": "search text"},
|
||||
"filter": {
|
||||
"type": "string",
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
{"type": "null"},
|
||||
],
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {"description": "tag"},
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert out["properties"]["query"]["type"] == "string"
|
||||
# anyOf with null collapsed to plain type
|
||||
assert "anyOf" not in out["properties"]["filter"]
|
||||
assert out["properties"]["filter"]["type"] == "string"
|
||||
assert out["properties"]["tags"]["items"]["type"] == "string"
|
||||
assert out["required"] == ["query"]
|
||||
|
||||
|
||||
class TestEnumNullStripping:
|
||||
"""Rule 3: Moonshot rejects null/empty-string inside enum arrays."""
|
||||
|
||||
def test_enum_null_value_stripped(self):
|
||||
"""enum containing Python None must have it removed for Moonshot."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db_type": {
|
||||
"type": "string",
|
||||
"enum": ["mysql", "postgresql", None],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
db_type = out["properties"]["db_type"]
|
||||
assert None not in db_type["enum"]
|
||||
assert "mysql" in db_type["enum"]
|
||||
assert "postgresql" in db_type["enum"]
|
||||
|
||||
def test_enum_empty_string_stripped(self):
|
||||
"""enum containing empty string '' must have it removed for Moonshot."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db_type": {
|
||||
"type": "string",
|
||||
"enum": ["mysql", "postgresql", ""],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
db_type = out["properties"]["db_type"]
|
||||
assert "" not in db_type["enum"]
|
||||
assert db_type["enum"] == ["mysql", "postgresql"]
|
||||
|
||||
def test_enum_all_null_becomes_no_enum(self):
|
||||
"""enum that only had null/empty values is dropped entirely."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"val": {
|
||||
"type": "string",
|
||||
"enum": [None, ""],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
assert "enum" not in out["properties"]["val"]
|
||||
|
||||
def test_dataslayer_db_type_after_mcp_normalize(self):
|
||||
"""Real-world: dataslayer db_type anyOf+enum after MCP normalization."""
|
||||
# This is the exact shape after _normalize_mcp_input_schema runs:
|
||||
# anyOf collapsed, but enum still has null + empty string
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"datasource": {"type": "string"},
|
||||
"db_type": {
|
||||
"enum": ["mysql", "mariadb", "postgresql", "sqlserver", "oracle", "", None],
|
||||
"type": "string",
|
||||
"nullable": True,
|
||||
"default": None,
|
||||
},
|
||||
},
|
||||
"required": ["datasource"],
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
db_type = out["properties"]["db_type"]
|
||||
assert "nullable" not in db_type, "nullable keyword must be stripped"
|
||||
assert None not in db_type["enum"]
|
||||
assert "" not in db_type["enum"]
|
||||
assert db_type["enum"] == ["mysql", "mariadb", "postgresql", "sqlserver", "oracle"]
|
||||
assert db_type["type"] == "string"
|
||||
|
||||
def test_enum_on_object_type_not_stripped(self):
|
||||
"""enum on non-scalar types (object) should NOT be touched."""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"enum": [{}, None],
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
# object-typed enum should pass through unchanged
|
||||
assert "enum" in out["properties"]["config"]
|
||||
|
||||
def test_anyof_collapse_still_runs_nullable_and_enum_cleanup(self):
|
||||
"""After anyOf collapses to a single non-null branch, the merged
|
||||
node must still have ``nullable`` stripped and null/empty-string
|
||||
values removed from enum — not skipped by the early anyOf return.
|
||||
"""
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db_type": {
|
||||
"anyOf": [
|
||||
{"enum": ["mysql", "postgresql", "", None]},
|
||||
{"type": "null"},
|
||||
],
|
||||
"nullable": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
out = sanitize_moonshot_tool_parameters(params)
|
||||
db_type = out["properties"]["db_type"]
|
||||
assert "anyOf" not in db_type
|
||||
assert "nullable" not in db_type, "nullable must be stripped after anyOf collapse"
|
||||
assert db_type["type"] == "string"
|
||||
assert db_type["enum"] == ["mysql", "postgresql"], \
|
||||
"null/empty enum values must be stripped after anyOf collapse"
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Tests for the non-stream stale-call detector context estimator.
|
||||
|
||||
Covers:
|
||||
- ``estimate_request_context_tokens`` for Chat Completions, Responses API,
|
||||
bare lists, and mixed-shape dicts.
|
||||
- ``AIAgent._compute_non_stream_stale_timeout`` with both legacy ``messages``
|
||||
list and full ``api_kwargs`` dicts.
|
||||
- The May 2026 default-base change (300s -> 90s) and the lowered
|
||||
context-tier ceilings (450/600 -> 150/240).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
|
||||
def _write_config(tmp_path: Path, body: str) -> None:
|
||||
hermes_home = tmp_path
|
||||
(hermes_home / "config.yaml").write_text(body or "{}\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _make_agent(tmp_path: Path, **overrides):
|
||||
from run_agent import AIAgent
|
||||
kwargs = dict(
|
||||
model="gpt-5.5",
|
||||
provider="openai-codex",
|
||||
api_key="sk-dummy",
|
||||
base_url="https://chatgpt.com/backend-api/codex",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
platform="cli",
|
||||
)
|
||||
kwargs.update(overrides)
|
||||
return AIAgent(**kwargs)
|
||||
|
||||
|
||||
# ── estimator ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_estimator_chat_completions_messages():
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
payload = {
|
||||
"model": "gpt-5.4",
|
||||
"messages": [
|
||||
{"role": "user", "content": "x" * 400},
|
||||
{"role": "assistant", "content": "y" * 400},
|
||||
],
|
||||
}
|
||||
# 800+ chars from messages -> ~200 tokens (char/4 estimate)
|
||||
assert estimate_request_context_tokens(payload) >= 200
|
||||
|
||||
|
||||
def test_estimator_responses_api_input():
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
payload = {
|
||||
"model": "gpt-5.5",
|
||||
"instructions": "i" * 1000,
|
||||
"input": "x" * 4000,
|
||||
"tools": [{"name": "t", "description": "d" * 200}],
|
||||
}
|
||||
# input(4000) + instructions(1000) + tools (~stringified) -> well over 1000 tokens
|
||||
tokens = estimate_request_context_tokens(payload)
|
||||
assert tokens >= 1200, f"Responses API estimator returned {tokens}"
|
||||
|
||||
|
||||
def test_estimator_responses_api_long_session_triggers_tier():
|
||||
"""A real long Codex session (large ``input``) should clear the 50k boundary."""
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
payload = {
|
||||
"model": "gpt-5.5",
|
||||
"input": "x" * 240_000, # ~60k tokens (240k chars / 4)
|
||||
"instructions": "s" * 4000,
|
||||
}
|
||||
assert estimate_request_context_tokens(payload) > 50_000
|
||||
|
||||
|
||||
def test_estimator_bare_list_back_compat():
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
messages = [
|
||||
{"role": "user", "content": "x" * 800},
|
||||
]
|
||||
assert estimate_request_context_tokens(messages) >= 200
|
||||
|
||||
|
||||
def test_estimator_empty_inputs():
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
assert estimate_request_context_tokens({}) == 0
|
||||
assert estimate_request_context_tokens([]) == 0
|
||||
assert estimate_request_context_tokens(None) == 0
|
||||
|
||||
|
||||
def test_estimator_unknown_dict_fallback():
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
payload = {"random_field": "z" * 400}
|
||||
assert estimate_request_context_tokens(payload) > 50
|
||||
|
||||
|
||||
# ── default base + tier scaling ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_default_base_is_90s(monkeypatch, tmp_path):
|
||||
"""Default base stale timeout dropped from 300s to 90s (May 2026)."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
_write_config(tmp_path, "")
|
||||
|
||||
agent = _make_agent(tmp_path)
|
||||
base, implicit = agent._resolved_api_call_stale_timeout_base()
|
||||
assert base == 90.0
|
||||
assert implicit is True
|
||||
|
||||
|
||||
def test_short_codex_request_uses_base_only(monkeypatch, tmp_path):
|
||||
"""Codex payload below 50k tokens -> default 90s base."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
_write_config(tmp_path, "")
|
||||
|
||||
agent = _make_agent(tmp_path)
|
||||
payload = {"model": "gpt-5.5", "input": "hi", "instructions": ""}
|
||||
assert agent._compute_non_stream_stale_timeout(payload) == 90.0
|
||||
|
||||
|
||||
def test_long_codex_request_bumps_to_50k_tier(monkeypatch, tmp_path):
|
||||
"""Codex payload > 50k tokens -> at least 150s."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
_write_config(tmp_path, "")
|
||||
|
||||
agent = _make_agent(tmp_path)
|
||||
payload = {"model": "gpt-5.5", "input": "x" * 240_000, "instructions": ""}
|
||||
timeout = agent._compute_non_stream_stale_timeout(payload)
|
||||
assert timeout >= 150.0
|
||||
assert timeout < 240.0
|
||||
|
||||
|
||||
def test_very_long_codex_request_bumps_to_100k_tier(monkeypatch, tmp_path):
|
||||
"""Codex payload > 100k tokens -> at least 240s."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
_write_config(tmp_path, "")
|
||||
|
||||
agent = _make_agent(tmp_path)
|
||||
payload = {"model": "gpt-5.5", "input": "x" * 500_000, "instructions": ""}
|
||||
assert agent._compute_non_stream_stale_timeout(payload) >= 240.0
|
||||
|
||||
|
||||
def test_chat_completions_long_messages_bumps_tier(monkeypatch, tmp_path):
|
||||
"""Chat Completions estimator still works for the legacy messages path."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
_write_config(tmp_path, "")
|
||||
|
||||
agent = _make_agent(
|
||||
tmp_path,
|
||||
provider="openai",
|
||||
base_url="https://api.openai.com/v1",
|
||||
model="gpt-5.4",
|
||||
)
|
||||
payload = {
|
||||
"model": "gpt-5.4",
|
||||
"messages": [{"role": "user", "content": "x" * 240_000}],
|
||||
}
|
||||
assert agent._compute_non_stream_stale_timeout(payload) >= 150.0
|
||||
|
||||
|
||||
def test_explicit_user_config_overrides_default(monkeypatch, tmp_path):
|
||||
"""If the user explicitly sets a stale_timeout, the new defaults don't apply."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
_write_config(tmp_path, """\
|
||||
providers:
|
||||
openai-codex:
|
||||
stale_timeout_seconds: 1800
|
||||
""")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
|
||||
import importlib
|
||||
from hermes_cli import timeouts as to_mod
|
||||
importlib.reload(to_mod)
|
||||
|
||||
agent = _make_agent(tmp_path)
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1800.0
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Tests for the Nous OAuth 401 actionable-guidance branch in
|
||||
``agent.conversation_loop.run_conversation``.
|
||||
|
||||
Source-inspection style (matches ``test_gemini_fast_fallback.py``): we assert
|
||||
that the guidance strings exist in the function body so that the user-facing
|
||||
hint cannot be silently removed by a future refactor.
|
||||
|
||||
Regression context: ashh hit a Nous 401 (OAuth token expired / portal said
|
||||
account out of credits) plus a model slug ``deepseek/deepseek-v4-flash:free``
|
||||
that's OpenRouter syntax, not a Nous catalog name. The previous guidance
|
||||
branch only covered ``openai-codex`` and ``xai-oauth``; ``nous`` fell through
|
||||
to a generic "Your API key was rejected... run hermes setup" message, which is
|
||||
the wrong advice for a pure-OAuth provider.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
|
||||
from agent import conversation_loop
|
||||
|
||||
|
||||
def test_nous_provider_is_in_oauth_401_set():
|
||||
"""The provider-set gate that selects OAuth-specific guidance must
|
||||
include ``nous`` alongside ``openai-codex`` and ``xai-oauth``.
|
||||
"""
|
||||
source = inspect.getsource(conversation_loop.run_conversation)
|
||||
|
||||
# Be flexible about set element ordering — assert all three are listed
|
||||
# near each other in the gating expression.
|
||||
assert "\"openai-codex\"" in source
|
||||
assert "\"xai-oauth\"" in source
|
||||
assert "\"nous\"" in source
|
||||
|
||||
# And the gate string itself must mention all three so future refactors
|
||||
# that split nous off into its own gate still get caught.
|
||||
needle = "_provider in {\"openai-codex\", \"xai-oauth\", \"nous\"}"
|
||||
assert needle in source, (
|
||||
"Expected nous to be co-gated with the other OAuth providers in the "
|
||||
"actionable-401-guidance branch of run_conversation."
|
||||
)
|
||||
|
||||
|
||||
def test_nous_401_guidance_strings_present():
|
||||
"""User-facing remediation strings for Nous OAuth 401s must exist."""
|
||||
source = inspect.getsource(conversation_loop.run_conversation)
|
||||
|
||||
# Must tell the user it's an OAuth token problem, NOT an API key problem
|
||||
# (Nous Portal has no API key path — auth_type=oauth_device_code only).
|
||||
assert "Nous Portal OAuth token was rejected" in source
|
||||
|
||||
# Must give the exact re-auth command, not a generic "hermes setup".
|
||||
assert "hermes auth add nous --type oauth" in source
|
||||
|
||||
# Must point at the portal so users can check account/credit status.
|
||||
assert "portal.nousresearch.com" in source
|
||||
|
||||
|
||||
def test_free_slug_hint_for_nous_provider():
|
||||
"""When the failing model slug ends with ``:free`` and the provider is
|
||||
``nous``, the guidance must flag that ``:free`` is OpenRouter syntax and
|
||||
suggest switching providers via ``/model openrouter:<slug>``.
|
||||
|
||||
Without this hint, users re-OAuth successfully and then hit the same 401
|
||||
on the next message because Nous Portal doesn't carry the OpenRouter
|
||||
free-tier slug.
|
||||
"""
|
||||
source = inspect.getsource(conversation_loop.run_conversation)
|
||||
|
||||
assert "endswith(\":free\")" in source
|
||||
assert "OpenRouter slug" in source
|
||||
assert "/model openrouter:" in source
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Tests for agent/nous_rate_guard.py — cross-session Nous Portal rate limit guard."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rate_guard_env(tmp_path, monkeypatch):
|
||||
"""Isolate rate guard state to a temp directory."""
|
||||
hermes_home = str(tmp_path / ".hermes")
|
||||
os.makedirs(hermes_home, exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", hermes_home)
|
||||
# Clear any cached module-level imports
|
||||
return hermes_home
|
||||
|
||||
|
||||
class TestRecordNousRateLimit:
|
||||
"""Test recording rate limit state."""
|
||||
|
||||
def test_records_with_header_reset(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
headers = {"x-ratelimit-reset-requests-1h": "1800"}
|
||||
record_nous_rate_limit(headers=headers)
|
||||
|
||||
path = _state_path()
|
||||
assert os.path.exists(path)
|
||||
with open(path) as f:
|
||||
state = json.load(f)
|
||||
assert state["reset_seconds"] == pytest.approx(1800, abs=2)
|
||||
assert state["reset_at"] > time.time()
|
||||
|
||||
def test_records_with_per_minute_header(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
headers = {"x-ratelimit-reset-requests": "45"}
|
||||
record_nous_rate_limit(headers=headers)
|
||||
|
||||
with open(_state_path()) as f:
|
||||
state = json.load(f)
|
||||
assert state["reset_seconds"] == pytest.approx(45, abs=2)
|
||||
|
||||
def test_records_with_retry_after_header(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
headers = {"retry-after": "60"}
|
||||
record_nous_rate_limit(headers=headers)
|
||||
|
||||
with open(_state_path()) as f:
|
||||
state = json.load(f)
|
||||
assert state["reset_seconds"] == pytest.approx(60, abs=2)
|
||||
|
||||
def test_prefers_hourly_over_per_minute(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
headers = {
|
||||
"x-ratelimit-reset-requests-1h": "1800",
|
||||
"x-ratelimit-reset-requests": "45",
|
||||
}
|
||||
record_nous_rate_limit(headers=headers)
|
||||
|
||||
with open(_state_path()) as f:
|
||||
state = json.load(f)
|
||||
# Should use the hourly value, not the per-minute one
|
||||
assert state["reset_seconds"] == pytest.approx(1800, abs=2)
|
||||
|
||||
def test_falls_back_to_error_context_reset_at(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
future_reset = time.time() + 900
|
||||
record_nous_rate_limit(
|
||||
headers=None,
|
||||
error_context={"reset_at": future_reset},
|
||||
)
|
||||
|
||||
with open(_state_path()) as f:
|
||||
state = json.load(f)
|
||||
assert state["reset_at"] == pytest.approx(future_reset, abs=1)
|
||||
|
||||
def test_falls_back_to_default_cooldown(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
record_nous_rate_limit(headers=None)
|
||||
|
||||
with open(_state_path()) as f:
|
||||
state = json.load(f)
|
||||
# Default is 300 seconds (5 minutes)
|
||||
assert state["reset_seconds"] == pytest.approx(300, abs=2)
|
||||
|
||||
def test_custom_default_cooldown(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
record_nous_rate_limit(headers=None, default_cooldown=120.0)
|
||||
|
||||
with open(_state_path()) as f:
|
||||
state = json.load(f)
|
||||
assert state["reset_seconds"] == pytest.approx(120, abs=2)
|
||||
|
||||
def test_creates_directory_if_missing(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, _state_path
|
||||
|
||||
record_nous_rate_limit(headers={"retry-after": "10"})
|
||||
assert os.path.exists(_state_path())
|
||||
|
||||
|
||||
class TestNousRateLimitRemaining:
|
||||
"""Test checking remaining rate limit time."""
|
||||
|
||||
def test_returns_none_when_no_file(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import nous_rate_limit_remaining
|
||||
|
||||
assert nous_rate_limit_remaining() is None
|
||||
|
||||
def test_returns_remaining_seconds_when_active(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit, nous_rate_limit_remaining
|
||||
|
||||
record_nous_rate_limit(headers={"x-ratelimit-reset-requests-1h": "600"})
|
||||
remaining = nous_rate_limit_remaining()
|
||||
assert remaining is not None
|
||||
assert 595 < remaining <= 605 # ~600 seconds, allowing for test execution time
|
||||
|
||||
def test_returns_none_when_expired(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import nous_rate_limit_remaining, _state_path
|
||||
|
||||
# Write an already-expired state
|
||||
state_dir = os.path.dirname(_state_path())
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
with open(_state_path(), "w") as f:
|
||||
json.dump({"reset_at": time.time() - 10, "recorded_at": time.time() - 100}, f)
|
||||
|
||||
assert nous_rate_limit_remaining() is None
|
||||
# File should be cleaned up
|
||||
assert not os.path.exists(_state_path())
|
||||
|
||||
def test_handles_corrupt_file(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import nous_rate_limit_remaining, _state_path
|
||||
|
||||
state_dir = os.path.dirname(_state_path())
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
with open(_state_path(), "w") as f:
|
||||
f.write("not valid json{{{")
|
||||
|
||||
assert nous_rate_limit_remaining() is None
|
||||
|
||||
|
||||
class TestClearNousRateLimit:
|
||||
"""Test clearing rate limit state."""
|
||||
|
||||
def test_clears_existing_file(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import (
|
||||
record_nous_rate_limit,
|
||||
clear_nous_rate_limit,
|
||||
nous_rate_limit_remaining,
|
||||
_state_path,
|
||||
)
|
||||
|
||||
record_nous_rate_limit(headers={"retry-after": "600"})
|
||||
assert nous_rate_limit_remaining() is not None
|
||||
|
||||
clear_nous_rate_limit()
|
||||
assert nous_rate_limit_remaining() is None
|
||||
assert not os.path.exists(_state_path())
|
||||
|
||||
def test_clear_when_no_file(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import clear_nous_rate_limit
|
||||
|
||||
# Should not raise
|
||||
clear_nous_rate_limit()
|
||||
|
||||
|
||||
class TestFormatRemaining:
|
||||
"""Test human-readable duration formatting."""
|
||||
|
||||
def test_seconds(self):
|
||||
from agent.nous_rate_guard import format_remaining
|
||||
|
||||
assert format_remaining(30) == "30s"
|
||||
|
||||
def test_minutes(self):
|
||||
from agent.nous_rate_guard import format_remaining
|
||||
|
||||
assert format_remaining(125) == "2m 5s"
|
||||
|
||||
def test_exact_minutes(self):
|
||||
from agent.nous_rate_guard import format_remaining
|
||||
|
||||
assert format_remaining(120) == "2m"
|
||||
|
||||
def test_hours(self):
|
||||
from agent.nous_rate_guard import format_remaining
|
||||
|
||||
assert format_remaining(3720) == "1h 2m"
|
||||
|
||||
|
||||
class TestParseResetSeconds:
|
||||
"""Test header parsing for reset times."""
|
||||
|
||||
def test_case_insensitive_headers(self, rate_guard_env):
|
||||
from agent.nous_rate_guard import _parse_reset_seconds
|
||||
|
||||
headers = {"X-Ratelimit-Reset-Requests-1h": "1200"}
|
||||
assert _parse_reset_seconds(headers) == 1200.0
|
||||
|
||||
def test_returns_none_for_empty_headers(self):
|
||||
from agent.nous_rate_guard import _parse_reset_seconds
|
||||
|
||||
assert _parse_reset_seconds(None) is None
|
||||
assert _parse_reset_seconds({}) is None
|
||||
|
||||
def test_ignores_zero_values(self):
|
||||
from agent.nous_rate_guard import _parse_reset_seconds
|
||||
|
||||
headers = {"x-ratelimit-reset-requests-1h": "0"}
|
||||
assert _parse_reset_seconds(headers) is None
|
||||
|
||||
def test_ignores_invalid_values(self):
|
||||
from agent.nous_rate_guard import _parse_reset_seconds
|
||||
|
||||
headers = {"x-ratelimit-reset-requests-1h": "not-a-number"}
|
||||
assert _parse_reset_seconds(headers) is None
|
||||
|
||||
|
||||
class TestAuxiliaryClientIntegration:
|
||||
"""Test that the auxiliary client respects the rate guard."""
|
||||
|
||||
def test_try_nous_skips_when_rate_limited(self, rate_guard_env, monkeypatch):
|
||||
from agent.nous_rate_guard import record_nous_rate_limit
|
||||
|
||||
# Record a rate limit
|
||||
record_nous_rate_limit(headers={"retry-after": "600"})
|
||||
|
||||
# Mock _read_nous_auth to return valid creds (would normally succeed)
|
||||
import agent.auxiliary_client as aux
|
||||
monkeypatch.setattr(aux, "_read_nous_auth", lambda: {
|
||||
"access_token": "test-token",
|
||||
"inference_base_url": "https://api.nous.test/v1",
|
||||
})
|
||||
|
||||
result = aux._try_nous()
|
||||
assert result == (None, None)
|
||||
|
||||
def test_try_nous_works_when_not_rate_limited(self, rate_guard_env, monkeypatch):
|
||||
import agent.auxiliary_client as aux
|
||||
|
||||
# No rate limit recorded — _try_nous should proceed normally
|
||||
# (will return None because no real creds, but won't be blocked
|
||||
# by the rate guard)
|
||||
monkeypatch.setattr(aux, "_read_nous_auth", lambda: None)
|
||||
result = aux._try_nous()
|
||||
assert result == (None, None)
|
||||
|
||||
|
||||
class TestIsGenuineNousRateLimit:
|
||||
"""Tell a real account-level 429 apart from an upstream-capacity 429.
|
||||
|
||||
Nous Portal multiplexes upstreams (DeepSeek, Kimi, MiMo, Hermes).
|
||||
A 429 from an upstream out of capacity should NOT trip the
|
||||
cross-session breaker; a real user-quota 429 should.
|
||||
"""
|
||||
|
||||
def test_exhausted_hourly_bucket_in_429_headers_is_genuine(self):
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
|
||||
headers = {
|
||||
"x-ratelimit-limit-requests-1h": "800",
|
||||
"x-ratelimit-remaining-requests-1h": "0",
|
||||
"x-ratelimit-reset-requests-1h": "3100",
|
||||
"x-ratelimit-limit-requests": "200",
|
||||
"x-ratelimit-remaining-requests": "198",
|
||||
"x-ratelimit-reset-requests": "40",
|
||||
}
|
||||
assert is_genuine_nous_rate_limit(headers=headers) is True
|
||||
|
||||
def test_exhausted_tokens_bucket_is_genuine(self):
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
|
||||
headers = {
|
||||
"x-ratelimit-limit-tokens": "800000",
|
||||
"x-ratelimit-remaining-tokens": "0",
|
||||
"x-ratelimit-reset-tokens": "45", # < 60s threshold -> not genuine
|
||||
"x-ratelimit-limit-tokens-1h": "8000000",
|
||||
"x-ratelimit-remaining-tokens-1h": "0",
|
||||
"x-ratelimit-reset-tokens-1h": "1800", # >= 60s threshold -> genuine
|
||||
}
|
||||
assert is_genuine_nous_rate_limit(headers=headers) is True
|
||||
|
||||
def test_healthy_headers_on_429_are_upstream_capacity(self):
|
||||
# Classic upstream-capacity symptom: Nous edge reports plenty of
|
||||
# headroom on every bucket, but returns 429 anyway because
|
||||
# upstream (DeepSeek / Kimi / ...) is out of capacity.
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
|
||||
headers = {
|
||||
"x-ratelimit-limit-requests": "200",
|
||||
"x-ratelimit-remaining-requests": "198",
|
||||
"x-ratelimit-reset-requests": "40",
|
||||
"x-ratelimit-limit-requests-1h": "800",
|
||||
"x-ratelimit-remaining-requests-1h": "750",
|
||||
"x-ratelimit-reset-requests-1h": "3100",
|
||||
"x-ratelimit-limit-tokens": "800000",
|
||||
"x-ratelimit-remaining-tokens": "790000",
|
||||
"x-ratelimit-reset-tokens": "40",
|
||||
"x-ratelimit-limit-tokens-1h": "8000000",
|
||||
"x-ratelimit-remaining-tokens-1h": "7800000",
|
||||
"x-ratelimit-reset-tokens-1h": "3100",
|
||||
}
|
||||
assert is_genuine_nous_rate_limit(headers=headers) is False
|
||||
|
||||
def test_bare_429_with_no_headers_is_upstream(self):
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
|
||||
assert is_genuine_nous_rate_limit(headers=None) is False
|
||||
assert is_genuine_nous_rate_limit(headers={}) is False
|
||||
assert is_genuine_nous_rate_limit(
|
||||
headers={"content-type": "application/json"}
|
||||
) is False
|
||||
|
||||
def test_exhausted_bucket_with_short_reset_is_not_genuine(self):
|
||||
# remaining == 0 but reset in < 60s: almost certainly a
|
||||
# secondary per-minute throttle that will clear immediately --
|
||||
# not worth tripping the cross-session breaker.
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
|
||||
headers = {
|
||||
"x-ratelimit-limit-requests": "200",
|
||||
"x-ratelimit-remaining-requests": "0",
|
||||
"x-ratelimit-reset-requests": "30",
|
||||
}
|
||||
assert is_genuine_nous_rate_limit(headers=headers) is False
|
||||
|
||||
def test_last_known_state_with_exhausted_bucket_triggers_genuine(self):
|
||||
# Headers on the 429 lack rate-limit info, but the previous
|
||||
# successful response already showed the hourly bucket
|
||||
# exhausted -- the 429 is almost certainly that limit
|
||||
# continuing.
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
from agent.rate_limit_tracker import parse_rate_limit_headers
|
||||
|
||||
prior_headers = {
|
||||
"x-ratelimit-limit-requests-1h": "800",
|
||||
"x-ratelimit-remaining-requests-1h": "0",
|
||||
"x-ratelimit-reset-requests-1h": "2000",
|
||||
"x-ratelimit-limit-requests": "200",
|
||||
"x-ratelimit-remaining-requests": "100",
|
||||
"x-ratelimit-reset-requests": "30",
|
||||
"x-ratelimit-limit-tokens": "800000",
|
||||
"x-ratelimit-remaining-tokens": "700000",
|
||||
"x-ratelimit-reset-tokens": "30",
|
||||
"x-ratelimit-limit-tokens-1h": "8000000",
|
||||
"x-ratelimit-remaining-tokens-1h": "7000000",
|
||||
"x-ratelimit-reset-tokens-1h": "2000",
|
||||
}
|
||||
last_state = parse_rate_limit_headers(prior_headers, provider="nous")
|
||||
assert is_genuine_nous_rate_limit(
|
||||
headers=None, last_known_state=last_state
|
||||
) is True
|
||||
|
||||
def test_last_known_state_all_healthy_stays_upstream(self):
|
||||
# Prior state was healthy; bare 429 arrives; should be treated
|
||||
# as upstream capacity.
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
from agent.rate_limit_tracker import parse_rate_limit_headers
|
||||
|
||||
prior_headers = {
|
||||
"x-ratelimit-limit-requests-1h": "800",
|
||||
"x-ratelimit-remaining-requests-1h": "750",
|
||||
"x-ratelimit-reset-requests-1h": "2000",
|
||||
"x-ratelimit-limit-requests": "200",
|
||||
"x-ratelimit-remaining-requests": "180",
|
||||
"x-ratelimit-reset-requests": "30",
|
||||
"x-ratelimit-limit-tokens": "800000",
|
||||
"x-ratelimit-remaining-tokens": "790000",
|
||||
"x-ratelimit-reset-tokens": "30",
|
||||
"x-ratelimit-limit-tokens-1h": "8000000",
|
||||
"x-ratelimit-remaining-tokens-1h": "7900000",
|
||||
"x-ratelimit-reset-tokens-1h": "2000",
|
||||
}
|
||||
last_state = parse_rate_limit_headers(prior_headers, provider="nous")
|
||||
assert is_genuine_nous_rate_limit(
|
||||
headers=None, last_known_state=last_state
|
||||
) is False
|
||||
|
||||
def test_none_last_state_and_no_headers_is_upstream(self):
|
||||
from agent.nous_rate_guard import is_genuine_nous_rate_limit
|
||||
|
||||
assert is_genuine_nous_rate_limit(
|
||||
headers=None, last_known_state=None
|
||||
) is False
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Tests for agent/onboarding.py — contextual first-touch hint helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import yaml
|
||||
|
||||
from agent.onboarding import (
|
||||
BUSY_INPUT_FLAG,
|
||||
OPENCLAW_RESIDUE_FLAG,
|
||||
TOOL_PROGRESS_FLAG,
|
||||
busy_input_hint_cli,
|
||||
busy_input_hint_gateway,
|
||||
detect_openclaw_residue,
|
||||
is_seen,
|
||||
mark_seen,
|
||||
openclaw_residue_hint_cli,
|
||||
tool_progress_hint_cli,
|
||||
tool_progress_hint_gateway,
|
||||
)
|
||||
|
||||
|
||||
class TestIsSeen:
|
||||
def test_empty_config_unseen(self):
|
||||
assert is_seen({}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_missing_onboarding_unseen(self):
|
||||
assert is_seen({"display": {}}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_onboarding_not_dict_unseen(self):
|
||||
assert is_seen({"onboarding": "nope"}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_seen_dict_missing_flag(self):
|
||||
assert is_seen({"onboarding": {"seen": {}}}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_seen_flag_true(self):
|
||||
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: True}}}
|
||||
assert is_seen(cfg, BUSY_INPUT_FLAG) is True
|
||||
|
||||
def test_seen_flag_falsy(self):
|
||||
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: False}}}
|
||||
assert is_seen(cfg, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_other_flags_isolated(self):
|
||||
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: True}}}
|
||||
assert is_seen(cfg, TOOL_PROGRESS_FLAG) is False
|
||||
|
||||
|
||||
class TestMarkSeen:
|
||||
def test_creates_missing_file_and_sets_flag(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_preserves_other_config(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"model": {"default": "claude-sonnet-4.6"},
|
||||
"display": {"skin": "default"},
|
||||
}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert loaded["model"]["default"] == "claude-sonnet-4.6"
|
||||
assert loaded["display"]["skin"] == "default"
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_preserves_other_seen_flags(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"onboarding": {"seen": {TOOL_PROGRESS_FLAG: True}},
|
||||
}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert loaded["onboarding"]["seen"][TOOL_PROGRESS_FLAG] is True
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_idempotent(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
first = cfg_path.read_text()
|
||||
|
||||
# Second call must be a no-op on-disk content (file may be touched,
|
||||
# but the YAML contents should be identical).
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
second = cfg_path.read_text()
|
||||
|
||||
assert yaml.safe_load(first) == yaml.safe_load(second)
|
||||
|
||||
def test_handles_non_dict_onboarding(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({"onboarding": "corrupted"}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_handles_non_dict_seen(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({"onboarding": {"seen": "corrupted"}}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
|
||||
class TestHintMessages:
|
||||
def test_busy_input_hint_gateway_interrupt(self):
|
||||
msg = busy_input_hint_gateway("interrupt")
|
||||
assert "/busy queue" in msg
|
||||
assert "interrupted" in msg.lower()
|
||||
|
||||
def test_busy_input_hint_gateway_queue(self):
|
||||
msg = busy_input_hint_gateway("queue")
|
||||
assert "/busy interrupt" in msg
|
||||
assert "queued" in msg.lower()
|
||||
|
||||
def test_busy_input_hint_gateway_steer(self):
|
||||
msg = busy_input_hint_gateway("steer")
|
||||
assert "/busy interrupt" in msg
|
||||
assert "/busy queue" in msg
|
||||
assert "steer" in msg.lower()
|
||||
|
||||
def test_busy_input_hint_cli_interrupt(self):
|
||||
msg = busy_input_hint_cli("interrupt")
|
||||
assert "/busy queue" in msg
|
||||
|
||||
def test_busy_input_hint_cli_queue(self):
|
||||
msg = busy_input_hint_cli("queue")
|
||||
assert "/busy interrupt" in msg
|
||||
|
||||
def test_busy_input_hint_cli_steer(self):
|
||||
msg = busy_input_hint_cli("steer")
|
||||
assert "/busy interrupt" in msg
|
||||
assert "/busy queue" in msg
|
||||
assert "steer" in msg.lower()
|
||||
|
||||
def test_tool_progress_hints_mention_verbose(self):
|
||||
assert "/verbose" in tool_progress_hint_gateway()
|
||||
assert "/verbose" in tool_progress_hint_cli()
|
||||
|
||||
def test_hints_are_not_empty(self):
|
||||
for hint in (
|
||||
busy_input_hint_gateway("queue"),
|
||||
busy_input_hint_gateway("interrupt"),
|
||||
busy_input_hint_gateway("steer"),
|
||||
busy_input_hint_cli("queue"),
|
||||
busy_input_hint_cli("interrupt"),
|
||||
busy_input_hint_cli("steer"),
|
||||
tool_progress_hint_gateway(),
|
||||
tool_progress_hint_cli(),
|
||||
):
|
||||
assert hint.strip()
|
||||
|
||||
|
||||
class TestRoundTrip:
|
||||
"""After mark_seen, is_seen on the re-loaded config must return True."""
|
||||
|
||||
def test_mark_then_is_seen(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert is_seen(loaded, BUSY_INPUT_FLAG) is True
|
||||
assert is_seen(loaded, TOOL_PROGRESS_FLAG) is False
|
||||
|
||||
def test_mark_both_flags_independently(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
mark_seen(cfg_path, TOOL_PROGRESS_FLAG)
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert is_seen(loaded, BUSY_INPUT_FLAG) is True
|
||||
assert is_seen(loaded, TOOL_PROGRESS_FLAG) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenClaw residue banner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDetectOpenclawResidue:
|
||||
def test_returns_true_when_openclaw_dir_present(self, tmp_path):
|
||||
(tmp_path / ".openclaw").mkdir()
|
||||
assert detect_openclaw_residue(home=tmp_path) is True
|
||||
|
||||
def test_returns_false_when_absent(self, tmp_path):
|
||||
assert detect_openclaw_residue(home=tmp_path) is False
|
||||
|
||||
def test_returns_false_when_path_is_a_file(self, tmp_path):
|
||||
# A stray file named ``.openclaw`` is NOT a workspace — skip the banner.
|
||||
(tmp_path / ".openclaw").write_text("oops")
|
||||
assert detect_openclaw_residue(home=tmp_path) is False
|
||||
|
||||
def test_default_home_does_not_crash(self):
|
||||
# Smoke: real $HOME lookup must not raise regardless of state.
|
||||
assert isinstance(detect_openclaw_residue(), bool)
|
||||
|
||||
|
||||
class TestOpenclawResidueHint:
|
||||
def test_hint_mentions_migrate_command(self):
|
||||
# `migrate` is the non-destructive path — should lead the banner.
|
||||
msg = openclaw_residue_hint_cli()
|
||||
assert "hermes claw migrate" in msg
|
||||
assert "~/.openclaw" in msg
|
||||
|
||||
def test_hint_mentions_cleanup_command(self):
|
||||
# `cleanup` is mentioned as the follow-up archive step.
|
||||
assert "hermes claw cleanup" in openclaw_residue_hint_cli()
|
||||
|
||||
def test_hint_warns_cleanup_breaks_openclaw(self):
|
||||
# Archiving the directory breaks OpenClaw for users still running it —
|
||||
# the banner must flag that side effect.
|
||||
msg = openclaw_residue_hint_cli().lower()
|
||||
assert "openclaw will stop working" in msg or "stop working" in msg
|
||||
|
||||
def test_hint_not_empty(self):
|
||||
assert openclaw_residue_hint_cli().strip()
|
||||
|
||||
|
||||
class TestOpenclawResidueSeenFlag:
|
||||
def test_flag_independent_of_other_flags(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert is_seen(loaded, OPENCLAW_RESIDUE_FLAG) is False
|
||||
|
||||
def test_flag_round_trips(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
assert mark_seen(cfg_path, OPENCLAW_RESIDUE_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert is_seen(loaded, OPENCLAW_RESIDUE_FLAG) is True
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Tests for OpenRouter response caching header injection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_or_headers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBuildOrHeaders:
|
||||
"""Test the build_or_headers() helper in agent/auxiliary_client.py."""
|
||||
|
||||
def test_base_attribution_always_present(self):
|
||||
"""Attribution headers must always be included regardless of cache setting."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": False})
|
||||
assert headers["HTTP-Referer"] == "https://hermes-agent.nousresearch.com"
|
||||
assert headers["X-Title"] == "Hermes Agent"
|
||||
assert headers["X-OpenRouter-Categories"] == "productivity,cli-agent"
|
||||
|
||||
def test_cache_enabled(self):
|
||||
"""When response_cache is True, X-OpenRouter-Cache header is set."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True})
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
|
||||
def test_cache_disabled(self):
|
||||
"""When response_cache is False, no cache header is sent."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": False})
|
||||
assert "X-OpenRouter-Cache" not in headers
|
||||
assert "X-OpenRouter-Cache-TTL" not in headers
|
||||
|
||||
def test_cache_disabled_by_default_empty_config(self):
|
||||
"""Empty config dict means no cache headers (response_cache defaults to False)."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={})
|
||||
assert "X-OpenRouter-Cache" not in headers
|
||||
|
||||
def test_ttl_default(self):
|
||||
"""Default TTL (300) is included when cache is enabled."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 300})
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "300"
|
||||
|
||||
def test_ttl_custom(self):
|
||||
"""Custom TTL values within range are sent."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 3600})
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "3600"
|
||||
|
||||
def test_ttl_max(self):
|
||||
"""Maximum TTL (86400) is accepted."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 86400})
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "86400"
|
||||
|
||||
def test_ttl_out_of_range_too_high(self):
|
||||
"""TTL above 86400 is silently ignored (no TTL header sent)."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 100000})
|
||||
assert "X-OpenRouter-Cache-TTL" not in headers
|
||||
# But cache is still enabled
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
|
||||
def test_ttl_out_of_range_zero(self):
|
||||
"""TTL of 0 is below minimum — no TTL header sent."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 0})
|
||||
assert "X-OpenRouter-Cache-TTL" not in headers
|
||||
|
||||
def test_ttl_negative(self):
|
||||
"""Negative TTL is ignored."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": -5})
|
||||
assert "X-OpenRouter-Cache-TTL" not in headers
|
||||
|
||||
def test_ttl_not_a_number(self):
|
||||
"""Non-numeric TTL is ignored."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": "five"})
|
||||
assert "X-OpenRouter-Cache-TTL" not in headers
|
||||
|
||||
def test_ttl_float_truncated(self):
|
||||
"""Float TTL values are truncated to int."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 600.7})
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "600"
|
||||
|
||||
def test_returns_fresh_dict(self):
|
||||
"""Each call returns a new dict so mutations don't leak."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
cfg = {"response_cache": True}
|
||||
h1 = build_or_headers(or_config=cfg)
|
||||
h2 = build_or_headers(or_config=cfg)
|
||||
assert h1 is not h2
|
||||
assert h1 == h2
|
||||
|
||||
def test_none_config_falls_back_to_load_config(self):
|
||||
"""When or_config is None, build_or_headers reads from load_config()."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
fake_cfg = {
|
||||
"openrouter": {"response_cache": True, "response_cache_ttl": 900},
|
||||
}
|
||||
with patch("hermes_cli.config.load_config", return_value=fake_cfg):
|
||||
headers = build_or_headers(or_config=None)
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "900"
|
||||
|
||||
def test_none_config_load_config_fails_gracefully(self):
|
||||
"""When load_config() fails, build_or_headers still returns base headers."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
with patch("hermes_cli.config.load_config", side_effect=RuntimeError("boom")):
|
||||
headers = build_or_headers(or_config=None)
|
||||
# Should have base attribution but no cache headers
|
||||
assert "HTTP-Referer" in headers
|
||||
assert "X-OpenRouter-Cache" not in headers
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Environment variable overrides
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestEnvVarOverrides:
|
||||
"""Test env var precedence over config.yaml for response caching."""
|
||||
|
||||
def test_env_enables_cache(self, monkeypatch):
|
||||
"""HERMES_OPENROUTER_CACHE=true enables cache even when config disables it."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", "true")
|
||||
headers = build_or_headers(or_config={"response_cache": False})
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
|
||||
def test_env_disables_cache(self, monkeypatch):
|
||||
"""HERMES_OPENROUTER_CACHE=false disables cache even when config enables it."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", "false")
|
||||
headers = build_or_headers(or_config={"response_cache": True})
|
||||
assert "X-OpenRouter-Cache" not in headers
|
||||
|
||||
@pytest.mark.parametrize("value", ["1", "true", "TRUE", "yes", "Yes", "on"])
|
||||
def test_truthy_values(self, monkeypatch, value):
|
||||
"""Various truthy strings enable caching."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", value)
|
||||
headers = build_or_headers(or_config={})
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
|
||||
@pytest.mark.parametrize("value", ["0", "false", "no", "off", "maybe", ""])
|
||||
def test_non_truthy_values(self, monkeypatch, value):
|
||||
"""Non-truthy strings do not enable caching (empty falls through to config)."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", value)
|
||||
# Empty string falls through to config; others are explicitly non-truthy
|
||||
if value == "":
|
||||
# Empty env var falls through to config default (False)
|
||||
headers = build_or_headers(or_config={"response_cache": False})
|
||||
else:
|
||||
headers = build_or_headers(or_config={"response_cache": True})
|
||||
assert "X-OpenRouter-Cache" not in headers
|
||||
|
||||
def test_env_ttl_overrides_config(self, monkeypatch):
|
||||
"""HERMES_OPENROUTER_CACHE_TTL overrides config TTL."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", "true")
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE_TTL", "1800")
|
||||
headers = build_or_headers(or_config={"response_cache_ttl": 300})
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "1800"
|
||||
|
||||
@pytest.mark.parametrize("ttl", ["0", "86401", "abc", "-1", "12.5"])
|
||||
def test_invalid_env_ttl_dropped(self, monkeypatch, ttl):
|
||||
"""Invalid TTL env values are ignored; cache still enabled without TTL."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", "1")
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE_TTL", ttl)
|
||||
headers = build_or_headers(or_config={})
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
assert "X-OpenRouter-Cache-TTL" not in headers
|
||||
|
||||
@pytest.mark.parametrize("ttl", ["1", "300", "86400"])
|
||||
def test_valid_env_ttl_boundaries(self, monkeypatch, ttl):
|
||||
"""Boundary TTL values (1, 300, 86400) are accepted."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE", "yes")
|
||||
monkeypatch.setenv("HERMES_OPENROUTER_CACHE_TTL", ttl)
|
||||
assert build_or_headers(or_config={})["X-OpenRouter-Cache-TTL"] == ttl
|
||||
|
||||
def test_no_env_vars_falls_through_to_config(self, monkeypatch):
|
||||
"""Without env vars, config.yaml controls behavior."""
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
|
||||
monkeypatch.delenv("HERMES_OPENROUTER_CACHE", raising=False)
|
||||
monkeypatch.delenv("HERMES_OPENROUTER_CACHE_TTL", raising=False)
|
||||
headers = build_or_headers(or_config={"response_cache": True, "response_cache_ttl": 600})
|
||||
assert headers["X-OpenRouter-Cache"] == "true"
|
||||
assert headers["X-OpenRouter-Cache-TTL"] == "600"
|
||||
|
||||
class TestDefaultConfig:
|
||||
"""Verify the openrouter config section is in DEFAULT_CONFIG."""
|
||||
|
||||
def test_openrouter_section_exists(self):
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
|
||||
assert "openrouter" in DEFAULT_CONFIG
|
||||
or_cfg = DEFAULT_CONFIG["openrouter"]
|
||||
assert or_cfg["response_cache"] is True
|
||||
assert or_cfg["response_cache_ttl"] == 300
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_openrouter_cache_status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCheckOpenrouterCacheStatus:
|
||||
"""Test the _check_openrouter_cache_status method on AIAgent."""
|
||||
|
||||
def _make_agent(self):
|
||||
"""Create a minimal AIAgent-like object with just the method under test."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
# Use object.__new__ to skip __init__, then set the attributes we need
|
||||
agent = object.__new__(AIAgent)
|
||||
agent._or_cache_hits = 0
|
||||
return agent
|
||||
|
||||
def test_hit_increments_counter(self):
|
||||
agent = self._make_agent()
|
||||
resp = SimpleNamespace(headers={"x-openrouter-cache-status": "HIT"})
|
||||
agent._check_openrouter_cache_status(resp)
|
||||
assert agent._or_cache_hits == 1
|
||||
# Second hit increments
|
||||
agent._check_openrouter_cache_status(resp)
|
||||
assert agent._or_cache_hits == 2
|
||||
|
||||
def test_miss_does_not_increment(self):
|
||||
agent = self._make_agent()
|
||||
resp = SimpleNamespace(headers={"x-openrouter-cache-status": "MISS"})
|
||||
agent._check_openrouter_cache_status(resp)
|
||||
assert getattr(agent, "_or_cache_hits", 0) == 0
|
||||
|
||||
def test_no_header_is_noop(self):
|
||||
agent = self._make_agent()
|
||||
resp = SimpleNamespace(headers={})
|
||||
agent._check_openrouter_cache_status(resp)
|
||||
assert getattr(agent, "_or_cache_hits", 0) == 0
|
||||
|
||||
def test_none_response_is_safe(self):
|
||||
agent = self._make_agent()
|
||||
agent._check_openrouter_cache_status(None) # no crash
|
||||
|
||||
def test_no_headers_attr_is_safe(self):
|
||||
agent = self._make_agent()
|
||||
agent._check_openrouter_cache_status(object()) # no crash
|
||||
|
||||
def test_case_insensitive(self):
|
||||
agent = self._make_agent()
|
||||
resp = SimpleNamespace(headers={"x-openrouter-cache-status": "hit"})
|
||||
agent._check_openrouter_cache_status(resp)
|
||||
assert agent._or_cache_hits == 1
|
||||
@@ -0,0 +1,990 @@
|
||||
"""Unit tests for the plugin LLM facade (``agent.plugin_llm``).
|
||||
|
||||
These tests exercise the trust gate, JSON parsing, schema validation,
|
||||
image input encoding, and the auxiliary-client invocation contract.
|
||||
The auxiliary client itself is stubbed via ``make_plugin_llm_for_test``
|
||||
so we don't hit real providers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.plugin_llm import (
|
||||
PluginLlm,
|
||||
PluginLlmCompleteResult,
|
||||
PluginLlmImageInput,
|
||||
PluginLlmStructuredResult,
|
||||
PluginLlmTextInput,
|
||||
PluginLlmTrustError,
|
||||
_build_structured_messages,
|
||||
_check_overrides,
|
||||
_coerce_allowlist,
|
||||
_parse_structured_text,
|
||||
_strip_code_fences,
|
||||
_TrustPolicy,
|
||||
make_plugin_llm_for_test,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fake_response(text: str, *, prompt: int = 4, completion: int = 6) -> SimpleNamespace:
|
||||
"""Build an OpenAI-shaped response with the given text + token usage."""
|
||||
return SimpleNamespace(
|
||||
choices=[
|
||||
SimpleNamespace(
|
||||
message=SimpleNamespace(content=text, role="assistant"),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens=prompt,
|
||||
completion_tokens=completion,
|
||||
total_tokens=prompt + completion,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _trusted_policy(plugin_id: str = "trusted-plugin", **overrides: Any) -> _TrustPolicy:
|
||||
defaults = dict(
|
||||
allow_provider_override=True,
|
||||
allowed_providers=None,
|
||||
allow_any_provider=True,
|
||||
allow_model_override=True,
|
||||
allowed_models=None,
|
||||
allow_any_model=True,
|
||||
allow_agent_id_override=True,
|
||||
allow_profile_override=True,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return _TrustPolicy(plugin_id=plugin_id, **defaults)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trust gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTrustGate:
|
||||
def test_default_policy_blocks_provider_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the provider"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider="anthropic",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_default_policy_blocks_model_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the model"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="claude-3-5-sonnet",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_default_policy_blocks_agent_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="non-default agent id"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model=None,
|
||||
requested_agent_id="ada",
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_default_policy_blocks_profile_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the auth profile"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile="work",
|
||||
)
|
||||
|
||||
def test_overrides_independent(self):
|
||||
"""Each override is gated independently — turning on
|
||||
``allow_model_override`` does NOT also grant provider override."""
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="model-only",
|
||||
allow_model_override=True,
|
||||
allow_any_model=True,
|
||||
)
|
||||
# model alone passes
|
||||
_, m, _, _ = _check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="gpt-4o",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert m == "gpt-4o"
|
||||
# provider alone is still denied
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the provider"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider="anthropic",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_provider_allowlist_rejects_non_listed(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_provider_override=True,
|
||||
allowed_providers=frozenset({"openrouter", "anthropic"}),
|
||||
allow_any_provider=False,
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="not in plugins.entries"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider="openai",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_provider_allowlist_accepts_listed_case_insensitively(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_provider_override=True,
|
||||
allowed_providers=frozenset({"openrouter"}),
|
||||
allow_any_provider=False,
|
||||
)
|
||||
p, _, _, _ = _check_overrides(
|
||||
policy,
|
||||
requested_provider="OpenRouter",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert p == "OpenRouter"
|
||||
|
||||
def test_model_allowlist_rejects_non_listed(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_model_override=True,
|
||||
allowed_models=frozenset({"openai/gpt-4o-mini"}),
|
||||
allow_any_model=False,
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="not in plugins.entries"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="anthropic/claude-3-opus",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_model_allowlist_accepts_listed_case_insensitively(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_model_override=True,
|
||||
allowed_models=frozenset({"openai/gpt-4o-mini"}),
|
||||
allow_any_model=False,
|
||||
)
|
||||
_, m, _, _ = _check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="OpenAI/GPT-4o-mini",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert m == "OpenAI/GPT-4o-mini"
|
||||
|
||||
def test_no_overrides_passes_through(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
result = _check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert result == (None, None, None, None)
|
||||
|
||||
def test_all_overrides_when_fully_trusted(self):
|
||||
policy = _trusted_policy()
|
||||
result = _check_overrides(
|
||||
policy,
|
||||
requested_provider="openrouter",
|
||||
requested_model="anthropic/claude-3-5-sonnet",
|
||||
requested_agent_id="ada",
|
||||
requested_profile="work",
|
||||
)
|
||||
assert result == ("openrouter", "anthropic/claude-3-5-sonnet", "ada", "work")
|
||||
|
||||
|
||||
class TestAllowlistCoercion:
|
||||
def test_missing_yields_none(self):
|
||||
ranges, allow_any = _coerce_allowlist(None)
|
||||
assert ranges is None
|
||||
assert allow_any is False
|
||||
|
||||
def test_list_of_strings(self):
|
||||
ranges, allow_any = _coerce_allowlist(["A", "B"])
|
||||
assert ranges == frozenset({"a", "b"})
|
||||
assert allow_any is False
|
||||
|
||||
def test_star_alone_means_any(self):
|
||||
ranges, allow_any = _coerce_allowlist(["*"])
|
||||
assert ranges == frozenset()
|
||||
assert allow_any is True
|
||||
|
||||
def test_star_plus_specific_keeps_specifics(self):
|
||||
ranges, allow_any = _coerce_allowlist(["*", "openrouter"])
|
||||
assert ranges == frozenset({"openrouter"})
|
||||
assert allow_any is True
|
||||
|
||||
def test_non_list_yields_none(self):
|
||||
ranges, allow_any = _coerce_allowlist("openrouter")
|
||||
assert ranges is None
|
||||
assert allow_any is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structured message building
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStructuredMessageBuilding:
|
||||
def test_text_only_input(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Extract the action items",
|
||||
inputs=[PluginLlmTextInput(text="meeting notes go here")],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["role"] == "user"
|
||||
parts = messages[0]["content"]
|
||||
assert parts[0]["type"] == "text"
|
||||
assert "Extract the action items" in parts[0]["text"]
|
||||
assert parts[1] == {"type": "text", "text": "meeting notes go here"}
|
||||
|
||||
def test_json_mode_adds_system_directive(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Summarise",
|
||||
inputs=[PluginLlmTextInput(text="content")],
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
assert messages[0]["role"] == "system"
|
||||
assert "JSON object" in messages[0]["content"]
|
||||
|
||||
def test_schema_name_appended_to_header(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Extract fields",
|
||||
inputs=[PluginLlmTextInput(text="data")],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name="action.items",
|
||||
system_prompt=None,
|
||||
)
|
||||
header = messages[0]["content"][0]["text"]
|
||||
assert "Schema name: action.items" in header
|
||||
|
||||
def test_image_bytes_encoded_as_data_url(self):
|
||||
png_bytes = b"\x89PNG\r\n\x1a\nfake"
|
||||
messages = _build_structured_messages(
|
||||
instructions="Read the image",
|
||||
inputs=[
|
||||
PluginLlmImageInput(data=png_bytes, mime_type="image/png"),
|
||||
PluginLlmTextInput(text="prefer printed text"),
|
||||
],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
parts = messages[0]["content"]
|
||||
assert parts[1]["type"] == "image_url"
|
||||
url = parts[1]["image_url"]["url"]
|
||||
assert url.startswith("data:image/png;base64,")
|
||||
decoded = base64.b64decode(url.split(",", 1)[1])
|
||||
assert decoded == png_bytes
|
||||
assert parts[2] == {"type": "text", "text": "prefer printed text"}
|
||||
|
||||
def test_image_url_passed_through(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Caption this",
|
||||
inputs=[PluginLlmImageInput(url="https://example.com/cat.jpg")],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
img_part = messages[0]["content"][1]
|
||||
assert img_part["type"] == "image_url"
|
||||
assert img_part["image_url"]["url"] == "https://example.com/cat.jpg"
|
||||
|
||||
def test_dict_inputs_normalized(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Test",
|
||||
inputs=[
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "url": "https://x.example/y.png"},
|
||||
],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
parts = messages[0]["content"]
|
||||
assert parts[1]["text"] == "hello"
|
||||
assert parts[2]["image_url"]["url"] == "https://x.example/y.png"
|
||||
|
||||
def test_invalid_input_block_rejected(self):
|
||||
with pytest.raises(ValueError, match="Unknown input block"):
|
||||
_build_structured_messages(
|
||||
instructions="Test",
|
||||
inputs=[{"type": "audio", "data": b""}],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJsonParsing:
|
||||
def test_strip_code_fences_with_json_label(self):
|
||||
assert _strip_code_fences('```json\n{"a":1}\n```') == '{"a":1}'
|
||||
|
||||
def test_strip_code_fences_without_label(self):
|
||||
assert _strip_code_fences("```\nfoo\n```") == "foo"
|
||||
|
||||
def test_strip_code_fences_no_fence(self):
|
||||
assert _strip_code_fences('{"a":1}') == '{"a":1}'
|
||||
|
||||
def test_parse_returns_text_when_not_json_mode(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='{"a": 1}', json_mode=False, json_schema=None
|
||||
)
|
||||
assert parsed is None
|
||||
assert ct == "text"
|
||||
|
||||
def test_parse_valid_json_with_json_mode(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='{"language": "French", "is_question": true}',
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
)
|
||||
assert parsed == {"language": "French", "is_question": True}
|
||||
assert ct == "json"
|
||||
|
||||
def test_parse_strips_code_fences_before_loading(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='Here you go:\n```json\n{"ok": true}\n```',
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
)
|
||||
assert parsed == {"ok": True}
|
||||
assert ct == "json"
|
||||
|
||||
def test_parse_returns_text_on_invalid_json(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text="not even close to json",
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
)
|
||||
assert parsed is None
|
||||
assert ct == "text"
|
||||
|
||||
def test_schema_validation_rejects_mismatch(self):
|
||||
pytest.importorskip("jsonschema")
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"language": {"type": "string"}},
|
||||
"required": ["language"],
|
||||
}
|
||||
with pytest.raises(ValueError, match="did not match schema"):
|
||||
_parse_structured_text(
|
||||
text='{"is_question": true}',
|
||||
json_mode=False,
|
||||
json_schema=schema,
|
||||
)
|
||||
|
||||
def test_schema_validation_accepts_match(self):
|
||||
pytest.importorskip("jsonschema")
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"language": {"type": "string"}},
|
||||
"required": ["language"],
|
||||
}
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='{"language": "French"}',
|
||||
json_mode=False,
|
||||
json_schema=schema,
|
||||
)
|
||||
assert parsed == {"language": "French"}
|
||||
assert ct == "json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end facade
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPluginLlmFacade:
|
||||
def test_complete_uses_active_model_by_default(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "auto", "default", _fake_response("Hello world.")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete([{"role": "user", "content": "hi"}])
|
||||
assert isinstance(result, PluginLlmCompleteResult)
|
||||
assert result.text == "Hello world."
|
||||
assert captured["provider_override"] is None
|
||||
assert captured["model_override"] is None
|
||||
assert captured["profile_override"] is None
|
||||
assert result.usage.input_tokens == 4
|
||||
assert result.usage.total_tokens == 10
|
||||
|
||||
def test_complete_rejects_provider_override_without_trust(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=lambda **_: ("x", "y", _fake_response("")),
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the provider"):
|
||||
llm.complete(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
provider="openrouter",
|
||||
)
|
||||
|
||||
def test_complete_rejects_model_override_without_trust(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=lambda **_: ("x", "y", _fake_response("")),
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the model"):
|
||||
llm.complete(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
model="anthropic/claude-3-opus",
|
||||
)
|
||||
|
||||
def test_complete_passes_through_trusted_overrides(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "anthropic", "claude-3-opus", _fake_response("ok")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_trusted_policy("my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
provider="anthropic",
|
||||
model="claude-3-opus",
|
||||
profile="work",
|
||||
agent_id="ada",
|
||||
temperature=0.0,
|
||||
max_tokens=128,
|
||||
timeout=10.0,
|
||||
purpose="extract",
|
||||
)
|
||||
# The recorded provider/model in the result come from the override,
|
||||
# since the stub caller echoed those values.
|
||||
assert result.provider == "anthropic"
|
||||
assert result.model == "claude-3-opus"
|
||||
assert captured["provider_override"] == "anthropic"
|
||||
assert captured["model_override"] == "claude-3-opus"
|
||||
assert captured["profile_override"] == "work"
|
||||
assert captured["temperature"] == 0.0
|
||||
assert captured["max_tokens"] == 128
|
||||
assert captured["timeout"] == 10.0
|
||||
|
||||
def test_complete_structured_returns_parsed_json(self):
|
||||
def fake_caller(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response(
|
||||
'{"language": "French", "is_question": true, "confidence": 0.99}'
|
||||
)
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete_structured(
|
||||
instructions="Detect language",
|
||||
input=[PluginLlmTextInput(text="Comment ça va?")],
|
||||
json_mode=True,
|
||||
)
|
||||
assert isinstance(result, PluginLlmStructuredResult)
|
||||
assert result.parsed == {
|
||||
"language": "French",
|
||||
"is_question": True,
|
||||
"confidence": 0.99,
|
||||
}
|
||||
assert result.content_type == "json"
|
||||
|
||||
def test_complete_structured_returns_text_on_unparseable_response(self):
|
||||
def fake_caller(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response("Sorry, I can't help with that.")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete_structured(
|
||||
instructions="Detect language",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
json_mode=True,
|
||||
)
|
||||
assert result.parsed is None
|
||||
assert result.content_type == "text"
|
||||
assert result.text.startswith("Sorry")
|
||||
|
||||
def test_complete_structured_validates_against_schema(self):
|
||||
pytest.importorskip("jsonschema")
|
||||
|
||||
def fake_caller(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response('{"unrelated": "field"}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"language": {"type": "string"}},
|
||||
"required": ["language"],
|
||||
}
|
||||
with pytest.raises(ValueError, match="did not match schema"):
|
||||
llm.complete_structured(
|
||||
instructions="Detect language",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
json_schema=schema,
|
||||
)
|
||||
|
||||
def test_complete_structured_requires_instructions(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=MagicMock(),
|
||||
)
|
||||
with pytest.raises(ValueError, match="non-empty instructions"):
|
||||
llm.complete_structured(
|
||||
instructions=" ",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
)
|
||||
|
||||
def test_complete_structured_requires_at_least_one_input(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=MagicMock(),
|
||||
)
|
||||
with pytest.raises(ValueError, match="at least one input"):
|
||||
llm.complete_structured(
|
||||
instructions="Extract",
|
||||
input=[],
|
||||
)
|
||||
|
||||
def test_complete_structured_emits_response_format_extra_body(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "openai", "gpt-4o", _fake_response('{"a": 1}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
schema = {"type": "object"}
|
||||
llm.complete_structured(
|
||||
instructions="Test",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
json_schema=schema,
|
||||
)
|
||||
rf = captured["extra_body"]["response_format"]
|
||||
assert rf["type"] == "json_schema"
|
||||
assert rf["json_schema"]["schema"] == schema
|
||||
|
||||
def test_complete_structured_with_image_passes_image_url_part(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "openai", "gpt-4o", _fake_response('{"caption": "ok"}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
png = b"fake-bytes"
|
||||
llm.complete_structured(
|
||||
instructions="Caption this",
|
||||
input=[PluginLlmImageInput(data=png, mime_type="image/png")],
|
||||
json_mode=True,
|
||||
)
|
||||
msgs = captured["messages"]
|
||||
user_msg = next(m for m in msgs if m["role"] == "user")
|
||||
image_parts = [p for p in user_msg["content"] if p.get("type") == "image_url"]
|
||||
assert len(image_parts) == 1
|
||||
assert image_parts[0]["image_url"]["url"].startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async surface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAsyncSurface:
|
||||
def test_acomplete_uses_async_caller(self):
|
||||
async def fake_async(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response("async hello")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
async_caller=fake_async,
|
||||
)
|
||||
|
||||
async def _run() -> PluginLlmCompleteResult:
|
||||
return await llm.acomplete([{"role": "user", "content": "hi"}])
|
||||
|
||||
result = asyncio.run(_run())
|
||||
assert result.text == "async hello"
|
||||
assert result.provider == "openai"
|
||||
|
||||
def test_acomplete_structured_parses_json(self):
|
||||
async def fake_async(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response('{"x": 42}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
async_caller=fake_async,
|
||||
)
|
||||
|
||||
async def _run() -> PluginLlmStructuredResult:
|
||||
return await llm.acomplete_structured(
|
||||
instructions="Extract x",
|
||||
input=[PluginLlmTextInput(text="data")],
|
||||
json_mode=True,
|
||||
)
|
||||
|
||||
result = asyncio.run(_run())
|
||||
assert result.parsed == {"x": 42}
|
||||
assert result.content_type == "json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config-driven trust gate (round-trip via plugins.entries.<id>.llm)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfigDrivenPolicy:
|
||||
def test_policy_loaded_from_yaml(self, tmp_path, monkeypatch):
|
||||
from agent.plugin_llm import _resolve_trust_policy
|
||||
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
"""
|
||||
plugins:
|
||||
entries:
|
||||
my-plugin:
|
||||
llm:
|
||||
allow_provider_override: true
|
||||
allowed_providers: [openrouter, anthropic]
|
||||
allow_model_override: true
|
||||
allowed_models:
|
||||
- openai/gpt-4o-mini
|
||||
- anthropic/claude-3-5-haiku
|
||||
allow_profile_override: false
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
from hermes_cli import config as _config_mod
|
||||
_config_mod._config_cache = None # type: ignore[attr-defined]
|
||||
|
||||
policy = _resolve_trust_policy("my-plugin")
|
||||
assert policy.allow_provider_override is True
|
||||
assert policy.allow_model_override is True
|
||||
assert policy.allow_profile_override is False
|
||||
assert policy.allowed_providers == frozenset({"openrouter", "anthropic"})
|
||||
assert policy.allowed_models == frozenset({
|
||||
"openai/gpt-4o-mini", "anthropic/claude-3-5-haiku",
|
||||
})
|
||||
|
||||
def test_missing_plugin_entry_yields_default_deny(self, tmp_path, monkeypatch):
|
||||
from agent.plugin_llm import _resolve_trust_policy
|
||||
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text("plugins: {}\n", encoding="utf-8")
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
from hermes_cli import config as _config_mod
|
||||
_config_mod._config_cache = None # type: ignore[attr-defined]
|
||||
|
||||
policy = _resolve_trust_policy("never-configured")
|
||||
assert policy.allow_provider_override is False
|
||||
assert policy.allow_model_override is False
|
||||
assert policy.allow_profile_override is False
|
||||
assert policy.allow_agent_id_override is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin context wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPluginContextIntegration:
|
||||
def test_ctx_llm_is_lazy_singleton(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="test-plugin", source="test", key="test-plugin")
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
first = ctx.llm
|
||||
second = ctx.llm
|
||||
assert first is second
|
||||
assert isinstance(first, PluginLlm)
|
||||
assert first._plugin_id == "test-plugin" # type: ignore[attr-defined]
|
||||
|
||||
def test_ctx_llm_uses_manifest_key_for_policy(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(
|
||||
name="bare-name", source="test", key="image_gen/openai"
|
||||
)
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
assert ctx.llm._plugin_id == "image_gen/openai" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Attribution (result.provider / result.model / audit log)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAttribution:
|
||||
"""Verifies that the result object and the audit log carry the real
|
||||
provider/model that ``call_llm`` ended up using, NOT the placeholder
|
||||
fallbacks ('auto', 'default') from earlier drafts."""
|
||||
|
||||
def test_explicit_overrides_recorded_when_no_response_model(self):
|
||||
from agent.plugin_llm import _resolve_attribution
|
||||
|
||||
# Response with no .model attribute — overrides win.
|
||||
response = SimpleNamespace(choices=[], usage=None)
|
||||
provider, model = _resolve_attribution(
|
||||
provider_override="openrouter",
|
||||
model_override="anthropic/claude-3-5-sonnet",
|
||||
response=response,
|
||||
)
|
||||
assert provider == "openrouter"
|
||||
assert model == "anthropic/claude-3-5-sonnet"
|
||||
|
||||
def test_response_model_wins_over_model_override(self):
|
||||
"""Providers often canonicalise the model name (e.g. ``gpt-4o``
|
||||
→ ``gpt-4o-2024-08-06``). Whatever they actually returned wins
|
||||
for the recorded model so the audit log reflects reality."""
|
||||
from agent.plugin_llm import _resolve_attribution
|
||||
|
||||
response = SimpleNamespace(model="gpt-4o-2024-08-06", choices=[])
|
||||
provider, model = _resolve_attribution(
|
||||
provider_override="openrouter",
|
||||
model_override="openai/gpt-4o",
|
||||
response=response,
|
||||
)
|
||||
assert model == "gpt-4o-2024-08-06"
|
||||
# Provider override is unaffected by response.model.
|
||||
assert provider == "openrouter"
|
||||
|
||||
def test_falls_back_to_main_provider_and_model_when_no_overrides(self, monkeypatch):
|
||||
"""When the plugin doesn't override anything, attribution
|
||||
reflects the user's active main provider/model rather than
|
||||
misleading placeholders."""
|
||||
from agent import plugin_llm
|
||||
import agent.auxiliary_client as ac
|
||||
|
||||
monkeypatch.setattr(ac, "_read_main_provider", lambda: "openrouter")
|
||||
monkeypatch.setattr(ac, "_read_main_model", lambda: "anthropic/claude-3-5-sonnet")
|
||||
|
||||
response = SimpleNamespace(choices=[]) # no .model attribute
|
||||
provider, model = plugin_llm._resolve_attribution(
|
||||
provider_override=None,
|
||||
model_override=None,
|
||||
response=response,
|
||||
)
|
||||
assert provider == "openrouter"
|
||||
assert model == "anthropic/claude-3-5-sonnet"
|
||||
|
||||
def test_response_model_used_even_when_no_overrides(self, monkeypatch):
|
||||
"""The provider's canonical model name should still flow through
|
||||
when no overrides are set."""
|
||||
from agent import plugin_llm
|
||||
import agent.auxiliary_client as ac
|
||||
|
||||
monkeypatch.setattr(ac, "_read_main_provider", lambda: "openrouter")
|
||||
monkeypatch.setattr(ac, "_read_main_model", lambda: "openai/gpt-4o")
|
||||
|
||||
response = SimpleNamespace(model="openai/gpt-4o-2024-08-06", choices=[])
|
||||
provider, model = plugin_llm._resolve_attribution(
|
||||
provider_override=None,
|
||||
model_override=None,
|
||||
response=response,
|
||||
)
|
||||
assert provider == "openrouter"
|
||||
assert model == "openai/gpt-4o-2024-08-06"
|
||||
|
||||
def test_placeholder_fallback_only_when_everything_is_empty(self, monkeypatch):
|
||||
"""If main_provider/main_model are unset AND there's no override
|
||||
AND the response has no .model, fall through to the safety
|
||||
placeholders so the result object never has empty strings."""
|
||||
from agent import plugin_llm
|
||||
import agent.auxiliary_client as ac
|
||||
|
||||
monkeypatch.setattr(ac, "_read_main_provider", lambda: "")
|
||||
monkeypatch.setattr(ac, "_read_main_model", lambda: "")
|
||||
|
||||
response = SimpleNamespace(choices=[])
|
||||
provider, model = plugin_llm._resolve_attribution(
|
||||
provider_override=None,
|
||||
model_override=None,
|
||||
response=response,
|
||||
)
|
||||
assert provider == "auto"
|
||||
assert model == "default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hook-mode integration (ctx.llm called from a post_tool_call callback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHookMode:
|
||||
"""The docs page promises ``ctx.llm`` works from inside lifecycle
|
||||
hooks. This exercises that path: register a ``post_tool_call``
|
||||
callback that calls ``ctx.llm.complete``, fire the hook through
|
||||
the real ``invoke_hook`` machinery, and check the call landed."""
|
||||
|
||||
def test_complete_works_from_post_tool_call_hook(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="hook-plugin", source="test", key="hook-plugin")
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
|
||||
# Replace ctx.llm with a stub that records what the hook called.
|
||||
captured: list = []
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.append(kwargs)
|
||||
return "openrouter", "openai/gpt-4o", _fake_response("rewrote it")
|
||||
|
||||
ctx._llm = make_plugin_llm_for_test( # type: ignore[attr-defined]
|
||||
plugin_id="hook-plugin",
|
||||
policy=_TrustPolicy(plugin_id="hook-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
|
||||
# Plugin registers a hook that runs ctx.llm.complete on every tool call.
|
||||
def rewrite_error_hook(*, tool_name, args, result, **_):
|
||||
if "Traceback" in (result or ""):
|
||||
rewritten = ctx.llm.complete(
|
||||
messages=[
|
||||
{"role": "system", "content": "Rewrite errors plainly."},
|
||||
{"role": "user", "content": result},
|
||||
],
|
||||
max_tokens=64,
|
||||
purpose="hook-plugin.rewrite",
|
||||
)
|
||||
# Real hook would return the rewritten text via
|
||||
# transform_tool_result; here we just capture for the assert.
|
||||
captured.append({"hook_returned": rewritten.text})
|
||||
|
||||
ctx.register_hook("post_tool_call", rewrite_error_hook)
|
||||
|
||||
# Fire the hook the same way the agent core does it.
|
||||
manager.invoke_hook(
|
||||
"post_tool_call",
|
||||
tool_name="terminal",
|
||||
args={"command": "boom"},
|
||||
result="Traceback (most recent call last):\n RuntimeError",
|
||||
)
|
||||
|
||||
# Verify ctx.llm.complete fired through the hook.
|
||||
assert len(captured) == 2 # one llm call + one hook return record
|
||||
llm_call = captured[0]
|
||||
assert "messages" in llm_call
|
||||
assert any("rewrite" in m.get("content", "").lower()
|
||||
for m in llm_call["messages"] if isinstance(m, dict))
|
||||
hook_record = captured[1]
|
||||
assert hook_record["hook_returned"] == "rewrote it"
|
||||
|
||||
def test_complete_works_from_post_tool_call_hook_when_async_caller_set(self):
|
||||
"""Hooks fired synchronously should still work with sync
|
||||
ctx.llm.complete even if other callsites use async."""
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="hook-async", source="test", key="hook-async")
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
|
||||
def fake_caller(**_):
|
||||
return "openrouter", "model-x", _fake_response("ok")
|
||||
|
||||
ctx._llm = make_plugin_llm_for_test( # type: ignore[attr-defined]
|
||||
plugin_id="hook-async",
|
||||
policy=_TrustPolicy(plugin_id="hook-async"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
|
||||
called: list = []
|
||||
|
||||
def hook(**kwargs):
|
||||
r = ctx.llm.complete(messages=[{"role": "user", "content": "x"}])
|
||||
called.append(r.text)
|
||||
|
||||
ctx.register_hook("post_tool_call", hook)
|
||||
manager.invoke_hook("post_tool_call", tool_name="x", args={}, result="y")
|
||||
assert called == ["ok"]
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Tests for agent.portal_tags — Nous Portal request tag contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_hermes_client_tag_includes_current_version():
|
||||
"""The client tag must reflect hermes_cli.__version__ verbatim."""
|
||||
from hermes_cli import __version__
|
||||
from agent.portal_tags import hermes_client_tag
|
||||
|
||||
assert hermes_client_tag() == f"client=hermes-client-v{__version__}"
|
||||
|
||||
|
||||
def test_hermes_client_tag_format():
|
||||
"""The client tag has the exact shape Nous Portal expects."""
|
||||
from agent.portal_tags import hermes_client_tag
|
||||
|
||||
tag = hermes_client_tag()
|
||||
assert tag.startswith("client=hermes-client-v")
|
||||
# No spaces, no commas — single tag value
|
||||
assert " " not in tag
|
||||
assert "," not in tag
|
||||
|
||||
|
||||
def test_nous_portal_tags_contains_product_and_client():
|
||||
"""Every Nous Portal request gets BOTH the product tag and the version tag."""
|
||||
from agent.portal_tags import hermes_client_tag, nous_portal_tags
|
||||
|
||||
tags = nous_portal_tags()
|
||||
assert "product=hermes-agent" in tags
|
||||
assert hermes_client_tag() in tags
|
||||
assert len(tags) == 2
|
||||
|
||||
|
||||
def test_nous_portal_tags_returns_fresh_list():
|
||||
"""Callers mutate the returned list; we must not share state across calls."""
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
|
||||
a = nous_portal_tags()
|
||||
a.append("client=test-mutation")
|
||||
b = nous_portal_tags()
|
||||
assert "client=test-mutation" not in b
|
||||
|
||||
|
||||
def test_auxiliary_client_nous_extra_body_uses_helper():
|
||||
"""auxiliary_client.NOUS_EXTRA_BODY must match the canonical helper output."""
|
||||
from agent.auxiliary_client import NOUS_EXTRA_BODY
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
|
||||
assert NOUS_EXTRA_BODY == {"tags": nous_portal_tags()}
|
||||
|
||||
|
||||
def test_nous_provider_profile_uses_helper():
|
||||
"""The Nous provider profile (main agent loop) must use the canonical tags."""
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
from providers import get_provider_profile
|
||||
|
||||
profile = get_provider_profile("nous")
|
||||
assert profile is not None
|
||||
body = profile.build_extra_body()
|
||||
assert body["tags"] == nous_portal_tags()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user