Initial import of NousResearch/hermes-agent
Deploy Site / deploy-vercel (push) Has been cancelled
Deploy Site / deploy-docs (push) Has been cancelled
Docker / shell lint / Lint Dockerfile (hadolint) (push) Has been cancelled
Docker / shell lint / Lint docker/ shell scripts (shellcheck) (push) Has been cancelled
Docker Build and Publish / build-amd64 (push) Has been cancelled
Docker Build and Publish / build-arm64 (push) Has been cancelled
Lint (ruff + ty) / ruff + ty diff (push) Has been cancelled
Lint (ruff + ty) / ruff enforcement (blocking) (push) Has been cancelled
Lint (ruff + ty) / Windows footguns (blocking) (push) Has been cancelled
Nix Lockfile Fix / auto-fix-main (push) Has been cancelled
Nix Lockfile Fix / fix (push) Has been cancelled
Nix / nix (macos-latest) (push) Has been cancelled
Nix / nix (ubuntu-latest) (push) Has been cancelled
OSV-Scanner / Scan lockfiles (push) Has been cancelled
Build Skills Index / build-index (push) Has been cancelled
Tests / test (1) (push) Has been cancelled
Tests / test (2) (push) Has been cancelled
Tests / test (3) (push) Has been cancelled
Tests / test (4) (push) Has been cancelled
Tests / test (5) (push) Has been cancelled
Tests / test (6) (push) Has been cancelled
Tests / e2e (push) Has been cancelled
uv.lock check / uv lock --check (push) Has been cancelled
Docker Build and Publish / merge (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
Tests / save-durations (push) Has been cancelled

This commit is contained in:
红尘
2026-05-31 09:36:58 +08:00
commit d73ff9b0fb
4188 changed files with 1614916 additions and 0 deletions
View File
View File
@@ -0,0 +1,273 @@
"""Behavior-parity check for the browser-provider plugin migration (#25214).
Spawns one subprocess per (version, scenario) cell — pinned to either
origin/main (legacy in-tree providers + class-instantiation lookup) or
this PR's worktree (plugin-based registry) via `sys.path[0]`. Each
subprocess clears all browser-related env vars + writes a config.yaml,
loads `tools.browser_tool._get_cloud_provider()`, and emits a reduced
"shape tuple" {is_local, provider_name, is_available} as JSON.
The parent process diffs the shapes per scenario. A diff means the
migration introduced an observable behaviour change vs origin/main —
which would be a real regression for users on the existing config keys.
Run from the PR worktree:
cd ~/.hermes/hermes-agent/.worktrees/browser-providers-plugin
python tests/plugins/browser/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
# Pin one path to current main, one to the PR worktree.
# ``REPO_ROOT`` is ``.../.worktrees/browser-providers-plugin``; the main
# checkout lives two levels up at ``~/.hermes/hermes-agent``.
MAIN_DIR = REPO_ROOT.parent.parent # ~/.hermes/hermes-agent
PR_DIR = REPO_ROOT # the worktree we're in
assert (MAIN_DIR / "tools" / "browser_tool.py").exists(), (
f"MAIN_DIR={MAIN_DIR} doesn't look like a hermes-agent checkout"
)
assert (PR_DIR / "tools" / "browser_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
# Reduced shape comparison — exact instance addresses obviously differ
# between subprocesses, so we compare the parts that matter for users.
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME for the config write.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear every browser-related env var so is_available() is deterministic.
for k in (
"BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", "BROWSERBASE_BASE_URL",
"BROWSER_USE_API_KEY", "BROWSER_USE_GATEWAY_URL",
"FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_BROWSER_TTL",
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
):
os.environ.pop(k, None)
# Apply per-scenario env (passed as JSON via argv[2]).
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
# Apply per-scenario config (passed as YAML body via argv[3]).
config_yaml = sys.argv[3]
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have any browser modules cached.
for name in list(sys.modules):
if name.startswith("tools.") or name.startswith("agent.") or name.startswith("plugins."):
sys.modules.pop(name, None)
from tools.browser_tool import _get_cloud_provider, _is_local_mode
provider = _get_cloud_provider()
# Pull the human-readable backend name via the API that exists on BOTH
# legacy (origin/main: CloudBrowserProvider.provider_name()) and the new
# ABC (BrowserProvider exposes provider_name() as a backward-compat alias
# returning display_name). Both shapes resolve to the same string —
# 'Browserbase' / 'Browser Use' / 'Firecrawl' — so we can compare safely.
provider_name = None
is_available = None
if provider is not None:
pn = getattr(provider, "provider_name", None)
if callable(pn):
provider_name = pn()
elif isinstance(pn, str):
provider_name = pn
is_conf = getattr(provider, "is_configured", None)
if callable(is_conf):
is_available = bool(is_conf())
shape = {
"is_local": _is_local_mode(),
"provider_name": provider_name,
"is_available": is_available,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
# (label, config.yaml body, extra env vars)
("no-config-no-env", "", {}),
("explicit-local-no-env", "browser:\n cloud_provider: local\n", {}),
(
"explicit-browserbase-no-creds",
"browser:\n cloud_provider: browserbase\n",
{},
),
(
"explicit-browserbase-with-creds",
"browser:\n cloud_provider: browserbase\n",
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
),
(
"explicit-browser-use-no-creds",
"browser:\n cloud_provider: browser-use\n",
{},
),
(
"explicit-browser-use-with-creds",
"browser:\n cloud_provider: browser-use\n",
{"BROWSER_USE_API_KEY": "k"},
),
(
"explicit-firecrawl-no-creds",
"browser:\n cloud_provider: firecrawl\n",
{},
),
(
"explicit-firecrawl-with-creds",
"browser:\n cloud_provider: firecrawl\n",
{"FIRECRAWL_API_KEY": "k"},
),
(
"no-config-bu-creds",
"",
{"BROWSER_USE_API_KEY": "k"},
),
(
"no-config-bb-creds",
"",
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
),
(
"no-config-both-creds",
"",
{
"BROWSER_USE_API_KEY": "k",
"BROWSERBASE_API_KEY": "x",
"BROWSERBASE_PROJECT_ID": "y",
},
),
(
"no-config-firecrawl-only",
"",
{"FIRECRAWL_API_KEY": "k"},
),
(
"no-config-firecrawl-and-bb",
"",
{
"FIRECRAWL_API_KEY": "k",
"BROWSERBASE_API_KEY": "x",
"BROWSERBASE_PROJECT_ID": "y",
},
),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
"""Run one (version, scenario) cell. Returns the shape dict."""
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
# Worktrees share the main repo's venv.
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
],
capture_output=True,
text=True,
timeout=30,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout,
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce_for_comparison(shape: dict) -> dict:
"""Reduce a shape dict to the parts that matter for user-visible parity.
We compare ``(is_local, provider_name, is_available)`` — the trio that
decides what the dispatcher does with each tool call. ``provider_name``
is the legacy ``provider_name()`` return value ('Browserbase' / 'Browser
Use' / 'Firecrawl'), which is identical between legacy and plugin
classes (the plugin's ``display_name`` matches the legacy
``provider_name()`` return).
"""
return {
"is_local": shape.get("is_local"),
"provider_name": shape.get("provider_name"),
"is_available": shape.get("is_available"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
failures: list[str] = []
errors: list[str] = []
for label, config_yaml, env in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce_for_comparison(main_shape)
pr_reduced = _reduce_for_comparison(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,379 @@
"""Plugin-side tests for the browser provider migration (PR #25214).
Covers:
- All three bundled plugins (browserbase, browser-use, firecrawl)
instantiate and self-report the expected ABC defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The browser_registry resolves an active provider in the documented
scenarios:
* explicit config wins ignoring availability (so dispatcher surfaces
a typed credentials error)
* legacy preference walk: browser-use → browserbase (filtered by
availability)
* firecrawl is NOT in the legacy walk — explicit-only
* unknown name falls through to auto-detect
* ``local`` short-circuits to None
These tests use *real* imports from the plugin modules — no mocking of
provider classes themselves — so the test catches drift in the ABC
interface, the registry, and the plugin glue layer simultaneously.
Mirrors ``tests/plugins/web/test_web_search_provider_plugins.py`` from
PR #25182.
"""
from __future__ import annotations
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _clear_browser_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every browser-provider env var so is_available() returns False."""
for k in (
"BROWSERBASE_API_KEY",
"BROWSERBASE_PROJECT_ID",
"BROWSERBASE_BASE_URL",
"BROWSER_USE_API_KEY",
"BROWSER_USE_GATEWAY_URL",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_BROWSER_TTL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_USER_TOKEN",
):
monkeypatch.delenv(k, raising=False)
def _ensure_plugins_loaded() -> None:
"""Idempotently load plugins so the registry is populated."""
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
# ---------------------------------------------------------------------------
# Per-test isolation
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each test starts with a clean browser-provider env."""
_clear_browser_env(monkeypatch)
# ---------------------------------------------------------------------------
# Bundled plugins register
# ---------------------------------------------------------------------------
class TestBundledPluginsRegister:
"""All three bundled browser plugins discover and register correctly."""
def test_all_three_plugins_present_in_registry(self) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import list_providers
names = sorted(p.name for p in list_providers())
assert names == ["browser-use", "browserbase", "firecrawl"]
@pytest.mark.parametrize(
"plugin_name,expected_display",
[
("browserbase", "Browserbase"),
("browser-use", "Browser Use"),
("firecrawl", "Firecrawl"),
],
)
def test_each_plugin_has_name_and_display_name(
self, plugin_name: str, expected_display: str
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None, f"plugin {plugin_name!r} not registered"
assert provider.name == plugin_name
assert provider.display_name == expected_display
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
"""``get_setup_schema()`` returns a dict the picker can consume."""
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
schema = provider.get_setup_schema()
assert isinstance(schema, dict)
assert "name" in schema
assert "env_vars" in schema
# Every cloud-browser plugin needs the agent-browser post-setup hook
# so the picker auto-installs the CLI on selection.
assert schema.get("post_setup") == "agent_browser"
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_each_plugin_implements_full_lifecycle(self, plugin_name: str) -> None:
"""The ABC's three lifecycle methods are all overridden."""
_ensure_plugins_loaded()
from agent.browser_provider import BrowserProvider
from agent.browser_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
# Each method must be a real override, not the ABC's NotImplementedError
# default — we check by comparing the function reference.
assert type(provider).create_session is not BrowserProvider.create_session
assert type(provider).close_session is not BrowserProvider.close_session
assert (
type(provider).emergency_cleanup is not BrowserProvider.emergency_cleanup
)
# ---------------------------------------------------------------------------
# is_available() behavior
# ---------------------------------------------------------------------------
class TestIsAvailable:
"""Each plugin's ``is_available()`` reflects env-var presence accurately."""
def test_browserbase_requires_both_api_key_and_project_id(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browserbase")
assert p is not None
assert p.is_available() is False
# API key alone is insufficient.
monkeypatch.setenv("BROWSERBASE_API_KEY", "key")
assert p.is_available() is False
# Both env vars set → available.
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
assert p.is_available() is True
def test_browserbase_project_id_alone_insufficient(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browserbase")
assert p is not None
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
assert p.is_available() is False
def test_browser_use_satisfied_by_api_key(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("browser-use")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("BROWSER_USE_API_KEY", "key")
assert p.is_available() is True
def test_firecrawl_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("FIRECRAWL_API_KEY", "key")
assert p.is_available() is True
# ---------------------------------------------------------------------------
# Registry resolution semantics
# ---------------------------------------------------------------------------
class TestRegistryResolution:
"""``_resolve()`` implements the documented three-rule precedence."""
def test_resolve_none_with_no_creds_returns_none(self) -> None:
"""No config, no env → local mode (None)."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
assert _resolve(None) is None
def test_explicit_local_returns_none(self) -> None:
"""``cloud_provider: local`` is a positive choice; short-circuits to None."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
assert _resolve("local") is None
def test_explicit_browserbase_returns_provider_even_when_unavailable(self) -> None:
"""Rule 1: explicit-config wins even when credentials are missing.
This is critical — the dispatcher needs to surface a typed
credentials error rather than silently switching backends.
"""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
provider = _resolve("browserbase")
assert provider is not None
assert provider.name == "browserbase"
assert provider.is_available() is False # confirms "ignoring availability"
def test_explicit_firecrawl_returns_provider_even_when_unavailable(self) -> None:
"""Firecrawl behaves the same as browserbase under explicit config."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
provider = _resolve("firecrawl")
assert provider is not None
assert provider.name == "firecrawl"
def test_explicit_unknown_falls_back_to_auto_detect(self) -> None:
"""Rule 1 miss: unknown name → fall through to legacy walk."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
# With no credentials anywhere, auto-detect should also fail.
assert _resolve("not-a-real-provider") is None
def test_legacy_walk_prefers_browser_use_over_browserbase(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rule 3: walk order is browser-use → browserbase."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
# Both available — browser-use should win.
monkeypatch.setenv("BROWSER_USE_API_KEY", "k1")
monkeypatch.setenv("BROWSERBASE_API_KEY", "k2")
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
provider = _resolve(None)
assert provider is not None
assert provider.name == "browser-use"
def test_legacy_walk_falls_through_to_browserbase(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rule 3: browser-use unavailable → browserbase picked."""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
monkeypatch.setenv("BROWSERBASE_API_KEY", "k")
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
provider = _resolve(None)
assert provider is not None
assert provider.name == "browserbase"
def test_firecrawl_not_in_legacy_walk_even_when_only_one_available(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Regression: firecrawl is NEVER auto-selected even when single-eligible.
Pre-PR-#25214, the dispatcher only auto-detected between Browser Use
and Browserbase; firecrawl was reachable solely via explicit
config. We preserve that gate because FIRECRAWL_API_KEY is shared
with the *web* firecrawl plugin — auto-routing a web-extract user
to a paid cloud browser would be a real behaviour regression.
"""
_ensure_plugins_loaded()
from agent.browser_registry import _resolve
monkeypatch.setenv("FIRECRAWL_API_KEY", "k")
# Only firecrawl is_available() — but it's not in the legacy walk.
assert _resolve(None) is None
# ---------------------------------------------------------------------------
# Legacy ABC backward-compat aliases (is_configured / provider_name)
# ---------------------------------------------------------------------------
class TestLegacyAbcAliases:
"""is_configured() and provider_name() delegate to the new API."""
@pytest.mark.parametrize(
"plugin_name",
["browserbase", "browser-use", "firecrawl"],
)
def test_is_configured_delegates_to_is_available(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider(plugin_name)
assert p is not None
assert p.is_configured() is p.is_available()
@pytest.mark.parametrize(
"plugin_name,expected_label",
[
("browserbase", "Browserbase"),
("browser-use", "Browser Use"),
("firecrawl", "Firecrawl"),
],
)
def test_provider_name_returns_display_name(
self, plugin_name: str, expected_label: str
) -> None:
_ensure_plugins_loaded()
from agent.browser_registry import get_provider
p = get_provider(plugin_name)
assert p is not None
assert p.provider_name() == expected_label
# ---------------------------------------------------------------------------
# Picker integration
# ---------------------------------------------------------------------------
class TestPickerIntegration:
"""`_plugin_browser_providers()` exposes all three plugins as picker rows."""
def test_picker_rows_match_registered_plugins(self) -> None:
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
rows = _plugin_browser_providers()
names = sorted(r.get("browser_provider") for r in rows)
assert names == ["browser-use", "browserbase", "firecrawl"]
def test_picker_rows_carry_post_setup_hook(self) -> None:
"""Every browser plugin row has post_setup='agent_browser' so
selecting it triggers the agent-browser CLI install."""
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
for row in _plugin_browser_providers():
assert row.get("post_setup") == "agent_browser", (
f"plugin row {row['browser_provider']!r} missing post_setup hook"
)
def test_picker_rows_carry_browser_plugin_name_marker(self) -> None:
"""`browser_plugin_name` matches `browser_provider` so downstream
code can route through the registry when it wants to."""
_ensure_plugins_loaded()
from hermes_cli.tools_config import _plugin_browser_providers
for row in _plugin_browser_providers():
assert row.get("browser_plugin_name") == row.get("browser_provider")
@@ -0,0 +1,755 @@
"""Tests for the bundled Nous dashboard-auth plugin.
Covers four shapes from Phase 4 of ``.hermes/plans/2026-05-21-dashboard-oauth-auth.md``:
1. Plugin entry-point registration gating (env var checks).
2. ``start_login`` shape (PKCE/state, authorize URL parameters).
3. ``complete_login`` httpx-mocked happy path + error mapping.
4. ``verify_session`` JWT verification — RSA keypair, audience/issuer pinning,
``agent_instance_id`` cross-check, ``oauth_contract_version`` tolerance.
Also exercises ``revoke_session`` (no-op) and ``refresh_session``
(unconditional ``RefreshExpiredError``).
All HTTP is mocked: nothing in this file talks to a real Portal.
"""
from __future__ import annotations
import base64
import hashlib
import json
import time
import urllib.parse
from typing import Any, Dict
from unittest.mock import MagicMock, patch
import httpx
import jwt
import pytest
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
import plugins.dashboard_auth.nous as nous_plugin
from hermes_cli.dashboard_auth import (
InvalidCodeError,
LoginStart,
ProviderError,
RefreshExpiredError,
Session,
assert_protocol_compliance,
)
# ---------------------------------------------------------------------------
# RSA keypair fixture (module-scope — keygen is slow)
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def rsa_keypair() -> Dict[str, Any]:
"""Generate an RS256 keypair + matching JWK for verify_session tests."""
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
private_pem = key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
).decode()
public_numbers = key.public_key().public_numbers()
def _b64url_uint(n: int) -> str:
length = (n.bit_length() + 7) // 8
return (
base64.urlsafe_b64encode(n.to_bytes(length, "big")).rstrip(b"=").decode()
)
jwk = {
"kty": "RSA",
"use": "sig",
"alg": "RS256",
"kid": "test-key-1",
"n": _b64url_uint(public_numbers.n),
"e": _b64url_uint(public_numbers.e),
}
return {"private_pem": private_pem, "jwk": jwk, "kid": jwk["kid"]}
# ---------------------------------------------------------------------------
# Token-mint helper
# ---------------------------------------------------------------------------
def _mint_token(
rsa_keypair: Dict[str, Any],
*,
iss: str = "https://portal.example.com",
aud: str = "agent:inst123",
sub: str = "usr_abc",
agent_instance_id: str | None = "inst123",
oauth_contract_version: Any = 1,
org_id: str | None = "org_xyz",
scope: str = "agent_dashboard:access",
ttl_seconds: int = 900,
extra_claims: Dict[str, Any] | None = None,
) -> str:
now = int(time.time())
claims = {
"iss": iss,
"aud": aud,
"sub": sub,
"iat": now,
"exp": now + ttl_seconds,
"scope": scope,
}
if agent_instance_id is not None:
claims["agent_instance_id"] = agent_instance_id
if oauth_contract_version is not None:
claims["oauth_contract_version"] = oauth_contract_version
if org_id is not None:
claims["org_id"] = org_id
if extra_claims:
claims.update(extra_claims)
return jwt.encode(
claims,
rsa_keypair["private_pem"],
algorithm="RS256",
headers={"kid": rsa_keypair["kid"]},
)
def _patched_jwks(provider: nous_plugin.NousDashboardAuthProvider, rsa_keypair):
"""Patch the provider's JWKS client to return our fixture key."""
fake_key = MagicMock()
fake_key.key = serialization.load_pem_private_key(
rsa_keypair["private_pem"].encode(), password=None
).public_key()
fake_client = MagicMock()
fake_client.get_signing_key_from_jwt.return_value = fake_key
provider._jwks_client = fake_client
# ---------------------------------------------------------------------------
# Provider construction
# ---------------------------------------------------------------------------
class TestConstruction:
def test_protocol_compliance(self):
assert_protocol_compliance(nous_plugin.NousDashboardAuthProvider)
def test_name_and_display(self):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst1", portal_url="https://portal.example.com"
)
assert p.name == "nous"
assert p.display_name == "Nous Research"
def test_extracts_agent_instance_id(self):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:abc-123", portal_url="https://portal.example.com"
)
assert p._agent_instance_id == "abc-123"
def test_strips_trailing_slash_from_portal_url(self):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:x", portal_url="https://portal.example.com/"
)
assert p._portal_url == "https://portal.example.com"
def test_rejects_malformed_client_id(self):
with pytest.raises(ValueError, match="agent:"):
nous_plugin.NousDashboardAuthProvider(
client_id="hermes-dashboard", portal_url="https://x"
)
# ---------------------------------------------------------------------------
# Plugin entry point: env-gated registration
# ---------------------------------------------------------------------------
class TestPluginRegister:
def test_skips_when_client_id_missing(self, monkeypatch):
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
# Skip reason is surfaced for the gate's fail-closed message.
assert "HERMES_DASHBOARD_OAUTH_CLIENT_ID" in nous_plugin.LAST_SKIP_REASON
def test_registers_with_default_portal_url_when_only_client_id_set(
self, monkeypatch
):
"""Phase 7 follow-up: HERMES_DASHBOARD_PORTAL_URL is optional —
defaults to the production Nous Portal. The user shouldn't have
to set it for the common production deployment path."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert isinstance(registered, nous_plugin.NousDashboardAuthProvider)
assert registered._portal_url == "https://portal.nousresearch.com"
# Skip reason cleared on successful registration.
assert nous_plugin.LAST_SKIP_REASON == ""
def test_skips_when_client_id_malformed(self, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "hermes-dashboard")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "https://p.example")
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
# Skip reason names the offending value + contract shape.
assert "agent:" in nous_plugin.LAST_SKIP_REASON
assert "hermes-dashboard" in nous_plugin.LAST_SKIP_REASON
def test_registers_with_explicit_portal_url(self, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "https://p.example")
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:inst1"
assert registered._portal_url == "https://p.example"
def test_strips_whitespace_from_env_vars(self, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", " agent:x ")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", " https://p.example ")
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
def test_empty_portal_url_env_uses_default(self, monkeypatch):
"""Explicit empty string still falls back to the production
default — same handling as 'unset' so an empty Fly secret can't
accidentally point the dashboard at nowhere."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "")
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._portal_url == "https://portal.nousresearch.com"
# ---------------------------------------------------------------------------
# Plugin entry point: config.yaml + env-override precedence
# ---------------------------------------------------------------------------
class TestConfigYamlSource:
"""``dashboard.oauth.{client_id,portal_url}`` in ``config.yaml`` is the
canonical surface for these settings. ``HERMES_DASHBOARD_OAUTH_CLIENT_ID``
and ``HERMES_DASHBOARD_PORTAL_URL`` are operator overrides that win when
set — this is the contract Fly.io's platform-secret injection relies on,
and the contract that lets local devs experiment without setting env
vars.
Each test pins exactly one tier of the precedence chain so a regression
that flips the order is caught:
env (when truthy) > config.yaml (when truthy) > plugin default
"""
@pytest.fixture
def patch_config(self, monkeypatch):
"""Yield a callable that replaces ``hermes_cli.config.load_config``
with a stub returning the given dict. Tests pass the intended
``dashboard.oauth`` block; the stub returns the wrapping structure."""
def _set(oauth_block: Dict[str, Any] | None) -> None:
cfg = {}
if oauth_block is not None:
cfg = {"dashboard": {"oauth": oauth_block}}
monkeypatch.setattr(
"hermes_cli.config.load_config", lambda: cfg
)
return _set
def test_config_yaml_only_client_id_registers(self, patch_config, monkeypatch):
"""No env var, only config.yaml — plugin reads from config and
registers successfully. This is the path Teknium's review pushed
for (".env is for secrets only")."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
patch_config({"client_id": "agent:from-config"})
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-config"
# Defaults to production portal URL when neither config nor env
# specifies one.
assert registered._portal_url == "https://portal.nousresearch.com"
def test_config_yaml_client_id_and_portal_url(self, patch_config, monkeypatch):
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
patch_config({
"client_id": "agent:from-config",
"portal_url": "https://staging.portal.example",
})
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-config"
assert registered._portal_url == "https://staging.portal.example"
def test_env_overrides_config_client_id(self, patch_config, monkeypatch):
"""Env wins. Critical for Fly.io: the Portal injects
HERMES_DASHBOARD_OAUTH_CLIENT_ID at deploy time and we MUST
honour it even if a stale config.yaml ships in the image."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:from-env")
patch_config({"client_id": "agent:from-config"})
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-env", (
"env var must override config.yaml — Fly secret injection "
"depends on this precedence"
)
def test_env_overrides_config_portal_url(self, patch_config, monkeypatch):
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:x")
monkeypatch.setenv(
"HERMES_DASHBOARD_PORTAL_URL", "https://env.portal.example",
)
patch_config({
"client_id": "agent:x",
"portal_url": "https://config.portal.example",
})
ctx = MagicMock()
nous_plugin.register(ctx)
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._portal_url == "https://env.portal.example"
def test_empty_env_string_does_not_shadow_config(
self, patch_config, monkeypatch
):
"""``HERMES_DASHBOARD_OAUTH_CLIENT_ID=`` (set but empty) is
common in CI/Fly when a secret is provisioned-but-not-populated.
It MUST NOT shadow a valid config.yaml value with an empty
string — operators would lose the gate."""
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "")
patch_config({"client_id": "agent:from-config"})
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_called_once()
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
assert registered._client_id == "agent:from-config"
def test_neither_source_skips_with_helpful_reason(
self, patch_config, monkeypatch
):
"""Neither env nor config.yaml set — skip with a reason that
mentions BOTH surfaces so operators don't guess wrong about
which one to populate."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
patch_config(None)
ctx = MagicMock()
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
# Old behaviour: skip reason mentions the env var.
assert "HERMES_DASHBOARD_OAUTH_CLIENT_ID" in nous_plugin.LAST_SKIP_REASON
# New behaviour: skip reason ALSO mentions the config.yaml path
# so the user knows it's a valid alternative.
assert "dashboard.oauth.client_id" in nous_plugin.LAST_SKIP_REASON, (
f"skip reason omits the config.yaml surface — operators "
f"won't know it exists. got: {nous_plugin.LAST_SKIP_REASON!r}"
)
def test_config_yaml_load_failure_falls_through_cleanly(
self, monkeypatch
):
"""If load_config() raises (e.g. malformed YAML, IOError), the
plugin must not crash — it falls through to the env-only path
and either succeeds (if env is set) or surfaces the standard
'not set' skip reason."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
def _broken_load():
raise OSError("config.yaml not readable")
monkeypatch.setattr(
"hermes_cli.config.load_config", _broken_load
)
ctx = MagicMock()
# Must not raise.
nous_plugin.register(ctx)
ctx.register_dashboard_auth_provider.assert_not_called()
def test_config_yaml_with_non_dict_oauth_section(
self, monkeypatch
):
"""cfg_get handles 'config has a string where a section was
expected' robustly. Verify the plugin inherits that resilience
so a malformed user config doesn't crash startup."""
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"dashboard": {"oauth": "wrong type"}},
)
ctx = MagicMock()
nous_plugin.register(ctx)
# Falls through to the no-env-and-no-config path.
ctx.register_dashboard_auth_provider.assert_not_called()
# ---------------------------------------------------------------------------
# start_login
# ---------------------------------------------------------------------------
class TestStartLogin:
@pytest.fixture
def provider(self):
return nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst1", portal_url="https://portal.example.com"
)
def test_returns_login_start(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert isinstance(result, LoginStart)
def test_redirect_url_targets_portal_authorize(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert result.redirect_url.startswith(
"https://portal.example.com/oauth/authorize?"
)
def test_authorize_url_has_required_params(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
assert params["response_type"] == "code"
assert params["client_id"] == "agent:inst1"
assert params["redirect_uri"] == "https://hermes.fly.dev/auth/callback"
assert params["scope"] == "agent_dashboard:access"
assert params["code_challenge_method"] == "S256"
assert "state" in params
assert "code_challenge" in params
def test_code_verifier_in_cookie_payload_43_to_128_chars(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert "hermes_session_pkce" in result.cookie_payload
pkce = result.cookie_payload["hermes_session_pkce"]
# Shape: ``state=…;verifier=…`` (matches stub-provider convention so
# the auth-route layer's parser works uniformly across providers).
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
verifier = parts["verifier"]
# RFC 7636 §4.1
assert 43 <= len(verifier) <= 128
def test_state_in_cookie_payload_matches_url_param(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
assert parts["state"] == params["state"]
def test_code_challenge_is_s256_of_verifier(self, provider):
result = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
parsed = urllib.parse.urlparse(result.redirect_url)
params = dict(urllib.parse.parse_qsl(parsed.query))
pkce = result.cookie_payload["hermes_session_pkce"]
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
verifier = parts["verifier"]
expected_challenge = (
base64.urlsafe_b64encode(
hashlib.sha256(verifier.encode("ascii")).digest()
)
.rstrip(b"=")
.decode()
)
assert params["code_challenge"] == expected_challenge
def test_two_calls_produce_different_state_and_verifier(self, provider):
a = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
b = provider.start_login(
redirect_uri="https://hermes.fly.dev/auth/callback"
)
assert a.cookie_payload["hermes_session_pkce"] != b.cookie_payload[
"hermes_session_pkce"
]
def test_rejects_non_http_scheme(self, provider):
with pytest.raises(ProviderError, match="http"):
provider.start_login(redirect_uri="ftp://x/auth/callback")
def test_rejects_http_with_non_localhost(self, provider):
with pytest.raises(ProviderError, match="localhost"):
provider.start_login(
redirect_uri="http://hermes.fly.dev/auth/callback"
)
def test_allows_http_localhost(self, provider):
# Should not raise.
provider.start_login(redirect_uri="http://localhost:8080/auth/callback")
provider.start_login(redirect_uri="http://127.0.0.1:8080/auth/callback")
def test_rejects_wrong_callback_path(self, provider):
with pytest.raises(ProviderError, match="/auth/callback"):
provider.start_login(redirect_uri="https://x.example/oauth/cb")
# ---------------------------------------------------------------------------
# complete_login (httpx mocked)
# ---------------------------------------------------------------------------
class TestCompleteLogin:
@pytest.fixture
def provider(self, rsa_keypair):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst123", portal_url="https://portal.example.com"
)
_patched_jwks(p, rsa_keypair)
return p
def _mock_post(self, status_code: int, body: Any, *, ctype: str = "application/json"):
resp = MagicMock(spec=httpx.Response)
resp.status_code = status_code
if isinstance(body, dict):
resp.text = json.dumps(body)
resp.json = MagicMock(return_value=body)
else:
resp.text = body
# _parse_json_body bails on non-application/json before .json()
# is called, but be safe for callers that pass a non-dict body
# with ctype=application/json.
resp.json = MagicMock(side_effect=ValueError("not json"))
resp.headers = {"content-type": ctype}
return resp
def test_happy_path_returns_session(self, provider, rsa_keypair):
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200, {"access_token": access_token, "token_type": "Bearer"}
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
session = provider.complete_login(
code="abc",
state="state-val",
code_verifier="vfy",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
assert isinstance(session, Session)
assert session.user_id == "usr_abc"
assert session.provider == "nous"
assert session.access_token == access_token
assert session.refresh_token == "" # contract V1
assert session.org_id == "org_xyz"
assert session.email == ""
assert session.display_name == ""
def test_400_raises_invalid_code(self, provider):
mock_resp = self._mock_post(400, {"error": "invalid_grant"})
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(InvalidCodeError, match="invalid_grant"):
provider.complete_login(
code="bad", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_500_raises_provider_error(self, provider):
mock_resp = self._mock_post(500, "internal server error", ctype="text/plain")
mock_resp.text = "internal server error"
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError, match="500"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_missing_access_token_raises(self, provider):
mock_resp = self._mock_post(200, {"token_type": "Bearer"})
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError, match="access_token"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_unexpected_token_type_raises(self, provider, rsa_keypair):
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200, {"access_token": access_token, "token_type": "DPoP"}
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
with pytest.raises(ProviderError, match="token_type"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_network_error_raises_provider_error(self, provider):
with patch(
"plugins.dashboard_auth.nous.httpx.post",
side_effect=httpx.ConnectError("conn refused"),
):
with pytest.raises(ProviderError, match="unreachable"):
provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
def test_captures_refresh_token_if_present_forward_compat(
self, provider, rsa_keypair
):
"""Forward-compat: contract V1 doesn't issue, but if a future Portal
does, we should preserve it in the Session for later use."""
access_token = _mint_token(rsa_keypair)
mock_resp = self._mock_post(
200,
{
"access_token": access_token,
"token_type": "Bearer",
"refresh_token": "rt-opaque",
},
)
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
session = provider.complete_login(
code="x", state="s", code_verifier="v",
redirect_uri="https://hermes.fly.dev/auth/callback",
)
assert session.refresh_token == "rt-opaque"
# ---------------------------------------------------------------------------
# verify_session
# ---------------------------------------------------------------------------
class TestVerifySession:
@pytest.fixture
def provider(self, rsa_keypair):
p = nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst123", portal_url="https://portal.example.com"
)
_patched_jwks(p, rsa_keypair)
return p
def test_happy_path_returns_session(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair)
session = provider.verify_session(access_token=token)
assert session is not None
assert session.user_id == "usr_abc"
assert session.org_id == "org_xyz"
def test_expired_token_returns_none(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, ttl_seconds=-1)
assert provider.verify_session(access_token=token) is None
def test_wrong_audience_raises_provider_error(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, aud="agent:other-instance")
with pytest.raises(ProviderError, match="verification failed"):
provider.verify_session(access_token=token)
def test_wrong_issuer_raises_provider_error(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, iss="https://evil.example")
with pytest.raises(ProviderError, match="verification failed"):
provider.verify_session(access_token=token)
def test_verification_failure_message_surfaces_token_claims(
self, provider, rsa_keypair
):
"""Operators need to see the actual iss/aud the token carries to debug
config drift between HERMES_DASHBOARD_PORTAL_URL/CLIENT_ID and Portal."""
token = _mint_token(rsa_keypair, iss="https://evil.example")
with pytest.raises(ProviderError) as excinfo:
provider.verify_session(access_token=token)
msg = str(excinfo.value)
# Both the observed (token) and expected (configured) values appear.
assert "'https://evil.example'" in msg
assert "'https://portal.example.com'" in msg # configured portal URL
def test_missing_sub_raises(self, provider, rsa_keypair):
# PyJWT's "require" set includes sub, so this surfaces as
# InvalidTokenError → ProviderError before we ever touch _session_from_claims.
token = _mint_token(rsa_keypair, sub="")
# Empty sub still encodes successfully; PyJWT's require check only
# asserts presence. Our own _session_from_claims rejects empty.
with pytest.raises(ProviderError, match="sub"):
provider.verify_session(access_token=token)
def test_agent_instance_id_mismatch_rejected(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, agent_instance_id="some-other-id")
with pytest.raises(ProviderError, match="agent_instance_id mismatch"):
provider.verify_session(access_token=token)
def test_agent_instance_id_missing_is_tolerated(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, agent_instance_id=None)
session = provider.verify_session(access_token=token)
assert session is not None
def test_contract_version_missing_warns_but_succeeds(
self, provider, rsa_keypair, caplog
):
import logging
token = _mint_token(rsa_keypair, oauth_contract_version=None)
with caplog.at_level(logging.WARNING, logger="plugins.dashboard_auth.nous"):
session = provider.verify_session(access_token=token)
assert session is not None
assert any(
"oauth_contract_version" in r.message for r in caplog.records
)
def test_contract_version_mismatch_rejected(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair, oauth_contract_version=2)
with pytest.raises(ProviderError, match="oauth_contract_version"):
provider.verify_session(access_token=token)
def test_jwks_unreachable_raises_provider_error(self, provider, rsa_keypair):
token = _mint_token(rsa_keypair)
# Replace the patched client so it raises.
bad_client = MagicMock()
bad_client.get_signing_key_from_jwt.side_effect = jwt.PyJWKClientError(
"fetch failed"
)
provider._jwks_client = bad_client
with pytest.raises(ProviderError, match="JWKS"):
provider.verify_session(access_token=token)
# ---------------------------------------------------------------------------
# refresh_session + revoke_session (V1 contract: trivial)
# ---------------------------------------------------------------------------
class TestRefreshAndRevoke:
@pytest.fixture
def provider(self):
return nous_plugin.NousDashboardAuthProvider(
client_id="agent:inst1", portal_url="https://portal.example.com"
)
def test_refresh_always_raises(self, provider):
with pytest.raises(RefreshExpiredError):
provider.refresh_session(refresh_token="anything")
def test_refresh_raises_even_with_empty_token(self, provider):
with pytest.raises(RefreshExpiredError):
provider.refresh_session(refresh_token="")
def test_revoke_is_noop(self, provider):
# Must not raise; returns None implicitly.
assert provider.revoke_session(refresh_token="anything") is None
assert provider.revoke_session(refresh_token="") is None
View File
@@ -0,0 +1,300 @@
"""Behavior-parity check for the image-gen FAL plugin migration (#26241).
Spawns one subprocess per (version, scenario) cell — pinned to either
``origin/main`` (legacy in-tree FAL fall-through + ``configured == "fal"``
skip in ``_dispatch_to_plugin_provider``) or this PR's worktree (FAL is
itself a plugin and the dispatcher routes every set provider through
the registry). Each subprocess clears all FAL-related env vars + writes
a ``config.yaml``, then asks the dispatcher how it would route an
``image_generate`` call. The emitted shape tuple is
``{dispatch_kind, provider_name, model}``:
* ``dispatch_kind`` ∈ ``{"legacy_fal", "plugin", "error", None}`` —
whether the call would go straight to the in-tree pipeline,
through ``_dispatch_to_plugin_provider``, raise an explicit
provider-not-registered error, or fall through silently.
* ``provider_name`` — when ``dispatch_kind == "plugin"``, the
resolved provider name. ``None`` otherwise.
* ``model`` — the resolved FAL model id when applicable.
The parent process diffs the shapes per scenario. A diff means the
migration introduced an observable behaviour change vs origin/main —
likely a real regression for users on the existing config keys.
Run from the PR worktree:
python tests/plugins/image_gen/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
# Pin one path to current main, one to the PR worktree.
# ``REPO_ROOT`` is ``.../.worktrees/<name>``; the main checkout lives
# two levels up. When running directly from a regular clone (no
# worktree), ``MAIN_DIR`` falls back to a sibling ``hermes-agent-main``
# checkout if one exists.
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "image_generation_tool.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "image_generation_tool.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "image_generation_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear FAL-related env so dispatch decisions are config-driven.
for k in (
"FAL_KEY", "FAL_QUEUE_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
"FAL_IMAGE_MODEL",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
import tools.image_generation_tool as image_tool
dispatch_kind = None
provider_name = None
model = None
error_text = None
try:
raw = image_tool._dispatch_to_plugin_provider("ping", "landscape")
if raw is None:
dispatch_kind = "legacy_fal"
else:
parsed = json.loads(raw) if isinstance(raw, str) else raw
if isinstance(parsed, dict):
if parsed.get("error_type") == "provider_not_registered":
dispatch_kind = "error"
error_text = parsed.get("error")
else:
dispatch_kind = "plugin"
provider_name = parsed.get("provider")
model = parsed.get("model")
else:
dispatch_kind = "unknown_payload"
if model is None:
# _resolve_fal_model still returns the active FAL model id even
# when dispatch goes to a non-FAL plugin — used for the diff
# only when applicable.
try:
model_id, _meta = image_tool._resolve_fal_model()
if dispatch_kind == "legacy_fal":
model = model_id
except Exception:
pass
except Exception as exc:
dispatch_kind = "exception"
error_text = repr(exc)
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"model": model,
"error_present": error_text is not None,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
# (label, config.yaml body, extra env vars)
("no-config-no-env", "", {}),
(
"explicit-fal-no-creds",
"image_gen:\n provider: fal\n",
{},
),
(
"explicit-fal-with-creds",
"image_gen:\n provider: fal\n",
{"FAL_KEY": "test-key"},
),
(
"explicit-fal-with-model",
"image_gen:\n provider: fal\n model: fal-ai/flux-2-pro\n",
{"FAL_KEY": "test-key"},
),
(
"explicit-typo-provider",
"image_gen:\n provider: not-a-real-backend\n",
{"FAL_KEY": "test-key"},
),
(
"managed-gateway-only",
"",
{
"TOOL_GATEWAY_DOMAIN": "nousresearch.com",
"TOOL_GATEWAY_USER_TOKEN": "nous-token",
},
),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
"""Reduce to the parts that matter for user-visible parity.
On origin/main, ``explicit-fal-*`` scenarios short-circuit to
``legacy_fal`` because of the ``configured == "fal"`` skip. On the
PR, those same scenarios route through the plugin and emit
``dispatch_kind == "plugin"`` with ``provider_name == "fal"``.
Both shapes are functionally equivalent — the plugin's ``generate()``
re-enters the same in-tree pipeline via ``_it`` indirection — but
we want the diff to be visible so reviewers can sign off on the
intentional behaviour delta.
"""
return {
"dispatch_kind": shape.get("dispatch_kind"),
"provider_name": shape.get("provider_name"),
"model": shape.get("model"),
"error_present": shape.get("error_present"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "explicit-fal-*" returns legacy_fal; on PR, plugin
# dispatch. That's the only acceptable diff — flag everything
# else as a regression.
legacy_to_plugin_fal = (
main_reduced.get("dispatch_kind") == "legacy_fal"
and pr_reduced.get("dispatch_kind") == "plugin"
and pr_reduced.get("provider_name") == "fal"
)
if legacy_to_plugin_fal:
print(f" [DIFF] {label}: legacy_fal → plugin (fal) — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"legacy_fal → plugin dispatch for explicit FAL paths."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,225 @@
#!/usr/bin/env python3
"""Tests for the FAL.ai image generation plugin.
The plugin is a thin registration adapter — actual FAL pipeline logic
lives in ``tools.image_generation_tool`` and is exercised by
``tests/tools/test_image_generation.py``. These tests focus on:
* the ``ImageGenProvider`` ABC surface (name, models, schema)
* call-time indirection (``_it`` resolution at ``generate()`` time so
``monkeypatch.setattr(image_tool, ...)`` keeps working)
* response shape stamping (provider/prompt/aspect_ratio/model)
"""
from __future__ import annotations
import json
from unittest.mock import MagicMock
# ---------------------------------------------------------------------------
# Provider surface
# ---------------------------------------------------------------------------
class TestFalImageGenProviderSurface:
def test_name(self):
from plugins.image_gen.fal import FalImageGenProvider
assert FalImageGenProvider().name == "fal"
def test_display_name(self):
from plugins.image_gen.fal import FalImageGenProvider
assert FalImageGenProvider().display_name == "FAL.ai"
def test_default_model_matches_legacy(self):
from plugins.image_gen.fal import FalImageGenProvider
from tools.image_generation_tool import DEFAULT_MODEL
assert FalImageGenProvider().default_model() == DEFAULT_MODEL
def test_list_models_uses_legacy_catalog(self):
from plugins.image_gen.fal import FalImageGenProvider
from tools.image_generation_tool import FAL_MODELS
provider = FalImageGenProvider()
models = provider.list_models()
ids = {m["id"] for m in models}
# Whatever FAL_MODELS ships, the provider mirrors verbatim.
assert ids == set(FAL_MODELS.keys())
# Spot-check the expected first-class fields are present.
for entry in models:
for field in ("id", "display", "speed", "strengths", "price"):
assert field in entry
def test_setup_schema_advertises_fal_key(self):
from plugins.image_gen.fal import FalImageGenProvider
schema = FalImageGenProvider().get_setup_schema()
assert schema["name"] == "FAL.ai"
assert schema["badge"] == "paid"
env_keys = {entry["key"] for entry in schema.get("env_vars", [])}
assert "FAL_KEY" in env_keys
class TestFalImageGenProviderAvailability:
def test_is_available_when_legacy_check_passes(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: True)
assert FalImageGenProvider().is_available() is True
def test_is_available_false_when_legacy_check_fails(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: False)
assert FalImageGenProvider().is_available() is False
def test_is_available_handles_legacy_exception(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
def _boom():
raise RuntimeError("config broke")
monkeypatch.setattr(image_tool, "check_fal_api_key", _boom)
# Picker must not propagate exceptions — show as "not available".
assert FalImageGenProvider().is_available() is False
# ---------------------------------------------------------------------------
# generate() — call-time indirection
# ---------------------------------------------------------------------------
class TestFalImageGenProviderGenerate:
def test_generate_delegates_to_legacy_image_generate_tool(self, monkeypatch):
"""Plugin must look up ``image_generate_tool`` at call time so
``monkeypatch.setattr(image_tool, "image_generate_tool", ...)``
takes effect."""
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
captured = {}
def fake_image_generate_tool(prompt, aspect_ratio, **kwargs):
captured["prompt"] = prompt
captured["aspect_ratio"] = aspect_ratio
captured["kwargs"] = kwargs
return json.dumps({"success": True, "image": "https://fake/image.png"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake_image_generate_tool)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
result = FalImageGenProvider().generate(
"a serene mountain landscape",
aspect_ratio="square",
seed=42,
)
assert captured["prompt"] == "a serene mountain landscape"
assert captured["aspect_ratio"] == "square"
assert captured["kwargs"] == {"seed": 42}
assert result["success"] is True
assert result["image"] == "https://fake/image.png"
# Stamped fields for the unified response shape
assert result["provider"] == "fal"
assert result["prompt"] == "a serene mountain landscape"
assert result["aspect_ratio"] == "square"
assert result["model"] == "fal-ai/flux-2/klein/9b"
def test_generate_invalid_aspect_ratio_is_coerced(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
seen_aspect = {}
def fake(prompt, aspect_ratio, **kwargs):
seen_aspect["v"] = aspect_ratio
return json.dumps({"success": True, "image": "x"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
FalImageGenProvider().generate("p", aspect_ratio="not-a-real-ratio")
# ``resolve_aspect_ratio`` clamps to landscape.
assert seen_aspect["v"] == "landscape"
def test_generate_passthrough_drops_none_kwargs(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
seen = {}
def fake(prompt, aspect_ratio, **kwargs):
seen.update(kwargs)
return json.dumps({"success": True, "image": "x"})
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
FalImageGenProvider().generate(
"p",
aspect_ratio="landscape",
seed=None,
num_images=2,
guidance_scale=None,
)
# ``None`` values must not be forwarded — they'd override the
# model's defaults inside the legacy payload builder.
assert "seed" not in seen
assert "guidance_scale" not in seen
assert seen.get("num_images") == 2
def test_generate_catches_exception_from_legacy(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
def boom(*args, **kwargs):
raise RuntimeError("FAL endpoint exploded")
monkeypatch.setattr(image_tool, "image_generate_tool", boom)
result = FalImageGenProvider().generate("p")
assert result["success"] is False
assert "FAL image generation failed" in result["error"]
assert result["error_type"] == "RuntimeError"
assert result["provider"] == "fal"
def test_generate_invalid_json_response(self, monkeypatch):
import tools.image_generation_tool as image_tool
from plugins.image_gen.fal import FalImageGenProvider
monkeypatch.setattr(image_tool, "image_generate_tool", lambda **kw: "not-json")
monkeypatch.setattr(image_tool, "_resolve_fal_model",
lambda: ("fal-ai/flux-2/klein/9b", {}))
result = FalImageGenProvider().generate("p")
assert result["success"] is False
assert "Invalid JSON" in result["error"]
assert result["provider"] == "fal"
# ---------------------------------------------------------------------------
# Registry wiring
# ---------------------------------------------------------------------------
class TestFalImageGenPluginRegistration:
def test_register_wires_provider_into_registry(self):
from plugins.image_gen.fal import FalImageGenProvider, register
ctx = MagicMock()
register(ctx)
ctx.register_image_gen_provider.assert_called_once()
(registered,), _ = ctx.register_image_gen_provider.call_args
assert isinstance(registered, FalImageGenProvider)
@@ -0,0 +1,625 @@
#!/usr/bin/env python3
"""Tests for Krea image generation provider."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _fake_api_key(monkeypatch):
"""Ensure KREA_API_KEY is set for all tests."""
monkeypatch.setenv("KREA_API_KEY", "test-key-12345")
def _completed_job(url: str = "https://krea.cdn/img.png") -> dict:
return {
"job_id": "00000000-0000-0000-0000-000000000abc",
"status": "completed",
"created_at": "2026-05-27T00:00:00Z",
"completed_at": "2026-05-27T00:00:30Z",
"result": {"urls": [url]},
}
def _submit_response(job_id: str = "00000000-0000-0000-0000-000000000abc"):
resp = MagicMock()
resp.status_code = 200
resp.raise_for_status = MagicMock()
resp.json.return_value = {
"job_id": job_id,
"status": "queued",
"created_at": "2026-05-27T00:00:00Z",
"completed_at": None,
"result": None,
}
return resp
def _poll_response(body: dict):
resp = MagicMock()
resp.status_code = 200
resp.raise_for_status = MagicMock()
resp.json.return_value = body
return resp
# ---------------------------------------------------------------------------
# Provider class tests
# ---------------------------------------------------------------------------
class TestKreaImageGenProvider:
def test_name(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().name == "krea"
def test_display_name(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().display_name == "Krea"
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("KREA_API_KEY", "sk-test")
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().is_available() is True
def test_is_available_without_key(self, monkeypatch):
monkeypatch.delenv("KREA_API_KEY", raising=False)
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().is_available() is False
def test_list_models(self):
from plugins.image_gen.krea import KreaImageGenProvider
models = KreaImageGenProvider().list_models()
ids = {m["id"] for m in models}
assert {"krea-2-medium", "krea-2-large"} <= ids
# Each entry carries the picker fields the registry expects.
for m in models:
assert m["display"]
assert m["speed"]
assert m["strengths"]
assert m["price"]
def test_default_model_is_medium(self):
from plugins.image_gen.krea import KreaImageGenProvider
assert KreaImageGenProvider().default_model() == "krea-2-medium"
def test_get_setup_schema(self):
from plugins.image_gen.krea import KreaImageGenProvider
schema = KreaImageGenProvider().get_setup_schema()
assert schema["name"] == "Krea"
assert schema["badge"] == "paid"
env_vars = schema["env_vars"]
assert len(env_vars) == 1
assert env_vars[0]["key"] == "KREA_API_KEY"
assert "krea.ai" in env_vars[0]["url"]
# ---------------------------------------------------------------------------
# Model resolution
# ---------------------------------------------------------------------------
class TestModelResolution:
def test_default(self):
from plugins.image_gen.krea import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "krea-2-medium"
assert meta["path"] == "medium"
def test_env_override_large(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
from plugins.image_gen.krea import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "krea-2-large"
assert meta["path"] == "large"
def test_env_override_unknown_falls_back_to_default(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-xxl-fake")
from plugins.image_gen.krea import _resolve_model
model_id, _ = _resolve_model()
assert model_id == "krea-2-medium"
def test_creativity_default(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity(None) == "medium"
def test_creativity_valid(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity("HIGH") == "high"
assert _resolve_creativity(" raw ") == "raw"
def test_creativity_invalid(self):
from plugins.image_gen.krea import _resolve_creativity
assert _resolve_creativity("ultra") == "medium"
# ---------------------------------------------------------------------------
# Generate — main flow
# ---------------------------------------------------------------------------
class TestGenerate:
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("KREA_API_KEY", raising=False)
from plugins.image_gen.krea import KreaImageGenProvider
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert "KREA_API_KEY" in result["error"]
assert result["error_type"] == "auth_required"
def test_empty_prompt(self):
from plugins.image_gen.krea import KreaImageGenProvider
result = KreaImageGenProvider().generate(prompt=" ")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_successful_generation(self):
"""Happy path: submit → one poll → completed → URL downloaded."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job("https://krea.cdn/result.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/krea_krea-2-medium_test.png"),
) as mock_save, \
patch("plugins.image_gen.krea.time.sleep"): # skip real waits
result = KreaImageGenProvider().generate(prompt="A cinematic lamp")
assert result["success"] is True
assert result["image"] == "/tmp/krea_krea-2-medium_test.png"
assert result["provider"] == "krea"
assert result["model"] == "krea-2-medium"
assert result["aspect_ratio"] == "landscape"
assert result["job_id"] == "00000000-0000-0000-0000-000000000abc"
assert result["resolution"] == "1K"
assert result["creativity"] == "medium"
# Submit hit the medium endpoint
post_url = mock_post.call_args[0][0]
assert post_url.endswith("/generate/image/krea/krea-2/medium")
# Poll hit /jobs/{job_id}
poll_url = mock_get.call_args[0][0]
assert "/jobs/00000000-0000-0000-0000-000000000abc" in poll_url
# URL was materialised once
mock_save.assert_called_once()
def test_large_model_routes_to_large_endpoint(self, monkeypatch):
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test")
post_url = mock_post.call_args[0][0]
assert post_url.endswith("/generate/image/krea/krea-2/large")
def test_aspect_ratio_mapping(self):
"""Hermes 'square' must map to Krea '1:1' in the wire payload."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test", aspect_ratio="square")
payload = mock_post.call_args.kwargs["json"]
assert payload["aspect_ratio"] == "1:1"
assert payload["resolution"] == "1K"
def test_auth_header(self):
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(prompt="test")
headers = mock_post.call_args.kwargs["headers"]
assert headers["Authorization"] == "Bearer test-key-12345"
assert headers["Content-Type"] == "application/json"
def test_passthrough_seed_styles_moodboards(self):
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
KreaImageGenProvider().generate(
prompt="test",
seed=42,
styles=[{"id": "lora-1", "strength": 0.7}],
moodboards=[{"url": "https://x.com/mood.png"}, {"url": "https://x.com/mood2.png"}],
image_style_references=[{"url": f"https://x.com/{i}.png"} for i in range(15)],
creativity="high",
)
payload = mock_post.call_args.kwargs["json"]
assert payload["seed"] == 42
assert payload["styles"] == [{"id": "lora-1", "strength": 0.7}]
assert len(payload["moodboards"]) == 1 # capped at 1
assert len(payload["image_style_references"]) == 10 # capped at 10
assert payload["creativity"] == "high"
def test_unknown_kwargs_ignored(self):
"""Forward-compat: unknown kwargs must not break generate()."""
from plugins.image_gen.krea import KreaImageGenProvider
submit = _submit_response()
poll = _poll_response(_completed_job())
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(
prompt="test",
fictional_param="should be ignored",
num_images=4,
)
assert result["success"] is True
# ---------------------------------------------------------------------------
# Generate — error paths
# ---------------------------------------------------------------------------
class TestGenerateErrors:
def test_submit_http_error(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
resp = req_lib.Response()
resp.status_code = 401
resp._content = b'{"error": {"message": "Invalid API key"}}'
resp.headers["Content-Type"] = "application/json"
resp.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=resp)
)
with patch("plugins.image_gen.krea.requests.post", return_value=resp):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "401" in result["error"]
assert "Invalid API key" in result["error"]
def test_submit_timeout(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
with patch(
"plugins.image_gen.krea.requests.post", side_effect=req_lib.Timeout()
):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "timeout"
def test_submit_connection_error(self):
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
with patch(
"plugins.image_gen.krea.requests.post",
side_effect=req_lib.ConnectionError("dns nope"),
):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "connection_error"
def test_submit_missing_job_id(self):
from plugins.image_gen.krea import KreaImageGenProvider
bad_submit = MagicMock()
bad_submit.status_code = 200
bad_submit.raise_for_status = MagicMock()
bad_submit.json.return_value = {"status": "queued"}
with patch("plugins.image_gen.krea.requests.post", return_value=bad_submit):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "invalid_response"
assert "job_id" in result["error"]
def test_job_failed(self):
from plugins.image_gen.krea import KreaImageGenProvider
failed = {
"job_id": "abc",
"status": "failed",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"error": "NSFW content"},
}
submit = _submit_response()
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(failed),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "NSFW" in result["error"]
def test_job_cancelled(self):
from plugins.image_gen.krea import KreaImageGenProvider
cancelled = {
"job_id": "abc",
"status": "cancelled",
"completed_at": "2026-05-27T00:01:00Z",
"result": {},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(cancelled),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "cancelled"
def test_completed_but_missing_urls(self):
from plugins.image_gen.krea import KreaImageGenProvider
completed_empty = {
"job_id": "abc",
"status": "completed",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"urls": []},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(completed_empty),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_url_download_failure_falls_back_to_bare_url(self):
"""Mirror of xAI behaviour — if local cache fails, return the URL."""
import requests as req_lib
from plugins.image_gen.krea import KreaImageGenProvider
url = "https://krea.cdn/expired-soon.png"
submit = _submit_response()
poll = _poll_response(_completed_job(url))
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
patch(
"plugins.image_gen.krea.save_url_image",
side_effect=req_lib.HTTPError("404"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert result["image"] == url
def test_polling_picks_up_completed_at_with_unknown_status(self):
"""``completed_at`` set + unrecognised pending status → still terminal."""
from plugins.image_gen.krea import KreaImageGenProvider
# Use a status value that is NOT in our terminal set ("intermediate-complete")
# but with completed_at populated — Krea's spec says completed_at is the
# canonical terminal marker.
oddball = {
"job_id": "abc",
"status": "intermediate-complete",
"completed_at": "2026-05-27T00:01:00Z",
"result": {"urls": ["https://krea.cdn/done.png"]},
}
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
return_value=_poll_response(oddball),
), \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
class TestPollRetryPolicy:
"""Polling fail-fast on permanent 4xx, retry on transient 5xx/429."""
def _http_error_response(self, status: int):
import requests as req_lib
resp = req_lib.Response()
resp.status_code = status
resp._content = b'{"error": "boom"}'
resp.headers["Content-Type"] = "application/json"
resp.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=resp)
)
return resp
def test_poll_fails_fast_on_401(self):
"""Auth failure mid-poll should not wait the 180s deadline."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(401)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "401" in result["error"]
# One call — no retry on permanent auth failure.
assert mock_get.call_count == 1
def test_poll_fails_fast_on_404(self):
"""Missing job (404) should surface immediately, not retry for 180s."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(404)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "404" in result["error"]
assert mock_get.call_count == 1
def test_poll_fails_fast_on_403(self):
"""Billing/permission failure (403) should not retry."""
from plugins.image_gen.krea import KreaImageGenProvider
bad_poll = self._http_error_response(403)
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is False
assert mock_get.call_count == 1
def test_poll_retries_on_503_then_succeeds(self):
"""Transient 5xx should retry and eventually surface a completion."""
from plugins.image_gen.krea import KreaImageGenProvider
flaky = self._http_error_response(503)
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
side_effect=[flaky, flaky, good],
) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert mock_get.call_count == 3
def test_poll_retries_on_429(self):
"""Rate-limit (429) is in the retryable set."""
from plugins.image_gen.krea import KreaImageGenProvider
rate_limited = self._http_error_response(429)
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
patch(
"plugins.image_gen.krea.requests.get",
side_effect=[rate_limited, good],
) as mock_get, \
patch(
"plugins.image_gen.krea.save_url_image",
return_value=Path("/tmp/x.png"),
), \
patch("plugins.image_gen.krea.time.sleep"):
result = KreaImageGenProvider().generate(prompt="test")
assert result["success"] is True
assert mock_get.call_count == 2
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
class TestRegistration:
def test_register(self):
from plugins.image_gen.krea import KreaImageGenProvider, register
mock_ctx = MagicMock()
register(mock_ctx)
mock_ctx.register_image_gen_provider.assert_called_once()
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
assert isinstance(provider, KreaImageGenProvider)
assert provider.name == "krea"
@@ -0,0 +1,236 @@
"""Tests for the bundled ``openai-codex`` image_gen plugin.
Mirrors ``test_openai_provider.py`` but targets the standalone
Codex/ChatGPT-OAuth-backed provider that uses the Responses
``image_generation`` tool path instead of the ``images.generate`` REST
endpoint.
"""
from __future__ import annotations
import importlib
from pathlib import Path
import pytest
# The plugin directory uses a hyphen, which is not a valid Python identifier
# for the dotted-import form. Load it via importlib so tests don't need to
# touch sys.path or rename the directory.
codex_plugin = importlib.import_module("plugins.image_gen.openai-codex")
# 1×1 transparent PNG — valid bytes for save_b64_image()
_PNG_HEX = (
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
"ae426082"
)
def _b64_png() -> str:
import base64
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
@pytest.fixture(autouse=True)
def _tmp_hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
@pytest.fixture
def provider(monkeypatch):
# Codex plugin is API-key-independent; clear it to make the test honest.
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
return codex_plugin.OpenAICodexImageGenProvider()
# ── Metadata ────────────────────────────────────────────────────────────────
class TestMetadata:
def test_name(self, provider):
assert provider.name == "openai-codex"
def test_display_name(self, provider):
assert provider.display_name == "OpenAI (Codex auth)"
def test_default_model(self, provider):
assert provider.default_model() == "gpt-image-2-medium"
def test_list_models_three_tiers(self, provider):
ids = [m["id"] for m in provider.list_models()]
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
def test_setup_schema_has_no_required_env_vars(self, provider):
schema = provider.get_setup_schema()
assert schema["env_vars"] == []
assert schema["badge"] == "free"
# ── Availability ────────────────────────────────────────────────────────────
class TestAvailability:
def test_unavailable_without_codex_token(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
def test_available_with_codex_token(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is True
def test_openai_api_key_alone_is_not_enough(self, monkeypatch):
# Codex plugin is intentionally orthogonal to the API-key plugin —
# the API key alone must NOT make it appear available.
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
# ── Generate ────────────────────────────────────────────────────────────────
class TestGenerate:
def test_returns_auth_error_without_codex_token(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_returns_invalid_argument_for_empty_prompt(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
result = provider.generate(" ")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_generate_uses_codex_stream_path(self, provider, monkeypatch, tmp_path):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: _b64_png())
result = provider.generate("a cat", aspect_ratio="landscape")
assert result["success"] is True
assert result["model"] == "gpt-image-2-medium"
assert result["provider"] == "openai-codex"
assert result["quality"] == "medium"
saved = Path(result["image"])
assert saved.exists()
assert saved.parent == tmp_path / "cache" / "images"
# Filename prefix differs from the API-key plugin so cache audits can
# tell the two backends apart.
assert saved.name.startswith("openai_codex_")
def test_codex_stream_request_shape(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
captured = {}
def _collect(token, *, prompt, size, quality):
captured.update(codex_plugin._build_responses_payload(
prompt=prompt,
size=size,
quality=quality,
))
return _b64_png()
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _collect)
result = provider.generate("a cat", aspect_ratio="portrait")
assert result["success"] is True
assert captured["model"] == "gpt-5.4"
assert captured["store"] is False
assert captured["input"][0]["type"] == "message"
assert captured["input"][0]["role"] == "user"
assert captured["input"][0]["content"][0]["type"] == "input_text"
assert captured["tool_choice"]["type"] == "allowed_tools"
assert captured["tool_choice"]["mode"] == "required"
assert captured["tool_choice"]["tools"] == [{"type": "image_generation"}]
tool = captured["tools"][0]
assert tool["type"] == "image_generation"
assert tool["model"] == "gpt-image-2"
assert tool["quality"] == "medium"
assert tool["size"] == "1024x1536"
assert tool["output_format"] == "png"
assert tool["background"] == "opaque"
assert tool["partial_images"] == 1
def test_partial_image_event_used_when_done_missing(self):
"""If output_item.done is missing, partial_image_b64 is accepted."""
payload = {
"type": "response.image_generation_call.partial_image",
"partial_image_b64": _b64_png(),
}
assert codex_plugin._extract_image_b64(payload) == _b64_png()
def test_sse_parser_handles_event_and_data_lines(self):
class _Response:
def iter_lines(self):
return iter([
"event: response.output_item.done",
'data: {"item": {"type": "image_generation_call", "result": "abc"}}',
"",
])
events = list(codex_plugin._iter_sse_json(_Response()))
assert events == [{
"type": "response.output_item.done",
"item": {"type": "image_generation_call", "result": "abc"},
}]
def test_final_response_sweep_recovers_image(self):
"""Completed response output is found by recursive payload scanning."""
payload = {
"type": "response.completed",
"response": {
"output": [{
"type": "image_generation_call",
"status": "completed",
"id": "ig_final",
"result": _b64_png(),
}],
},
}
assert codex_plugin._extract_image_b64(payload) == _b64_png()
def test_empty_response_returns_error(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: None)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_stream_exception_returns_api_error(self, provider, monkeypatch):
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
def _boom(*args, **kwargs):
raise RuntimeError("cloudflare 403")
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _boom)
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "cloudflare 403" in result["error"]
# ── Plugin entry point ──────────────────────────────────────────────────────
class TestRegistration:
def test_register_calls_register_image_gen_provider(self):
registered = []
class _Ctx:
def register_image_gen_provider(self, prov):
registered.append(prov)
codex_plugin.register(_Ctx())
assert len(registered) == 1
assert registered[0].name == "openai-codex"
@@ -0,0 +1,271 @@
"""Tests for the bundled OpenAI image_gen plugin (gpt-image-2, three tiers)."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import plugins.image_gen.openai as openai_plugin
# 1×1 transparent PNG — valid bytes for save_b64_image()
_PNG_HEX = (
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
"ae426082"
)
def _b64_png() -> str:
import base64
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
def _fake_response(*, b64=None, url=None, revised_prompt=None):
item = SimpleNamespace(b64_json=b64, url=url, revised_prompt=revised_prompt)
return SimpleNamespace(data=[item])
@pytest.fixture(autouse=True)
def _tmp_hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
@pytest.fixture
def provider(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
return openai_plugin.OpenAIImageGenProvider()
def _patched_openai(fake_client: MagicMock):
fake_openai = MagicMock()
fake_openai.OpenAI.return_value = fake_client
return patch.dict("sys.modules", {"openai": fake_openai})
# ── Metadata ────────────────────────────────────────────────────────────────
class TestMetadata:
def test_name(self, provider):
assert provider.name == "openai"
def test_default_model(self, provider):
assert provider.default_model() == "gpt-image-2-medium"
def test_list_models_three_tiers(self, provider):
ids = [m["id"] for m in provider.list_models()]
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
def test_catalog_entries_have_display_speed_strengths(self, provider):
for entry in provider.list_models():
assert entry["display"].startswith("GPT Image 2")
assert entry["speed"]
assert entry["strengths"]
# ── Availability ────────────────────────────────────────────────────────────
class TestAvailability:
def test_no_api_key_unavailable(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
assert openai_plugin.OpenAIImageGenProvider().is_available() is False
def test_api_key_set_available(self, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test")
assert openai_plugin.OpenAIImageGenProvider().is_available() is True
# ── Model resolution ────────────────────────────────────────────────────────
class TestModelResolution:
def test_default_is_medium(self):
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-medium"
assert meta["quality"] == "medium"
def test_env_var_override(self, monkeypatch):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "gpt-image-2-high")
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-high"
assert meta["quality"] == "high"
def test_env_var_unknown_falls_back(self, monkeypatch):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "bogus-tier")
model_id, _ = openai_plugin._resolve_model()
assert model_id == openai_plugin.DEFAULT_MODEL
def test_config_openai_model(self, tmp_path):
import yaml
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"openai": {"model": "gpt-image-2-low"}}})
)
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-low"
assert meta["quality"] == "low"
def test_config_top_level_model(self, tmp_path):
"""``image_gen.model: gpt-image-2-high`` also works (top-level)."""
import yaml
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"image_gen": {"model": "gpt-image-2-high"}})
)
model_id, meta = openai_plugin._resolve_model()
assert model_id == "gpt-image-2-high"
assert meta["quality"] == "high"
# ── Generate ────────────────────────────────────────────────────────────────
class TestGenerate:
def test_empty_prompt_rejected(self, provider):
result = provider.generate("", aspect_ratio="square")
assert result["success"] is False
assert result["error_type"] == "invalid_argument"
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
result = openai_plugin.OpenAIImageGenProvider().generate("a cat")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_b64_saves_to_cache(self, provider, tmp_path):
png_bytes = bytes.fromhex(_PNG_HEX)
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
result = provider.generate("a cat", aspect_ratio="landscape")
assert result["success"] is True
assert result["model"] == "gpt-image-2-medium"
assert result["aspect_ratio"] == "landscape"
assert result["provider"] == "openai"
assert result["quality"] == "medium"
saved = Path(result["image"])
assert saved.exists()
assert saved.parent == tmp_path / "cache" / "images"
assert saved.read_bytes() == png_bytes
call_kwargs = fake_client.images.generate.call_args.kwargs
# All tiers hit the single underlying API model.
assert call_kwargs["model"] == "gpt-image-2"
assert call_kwargs["quality"] == "medium"
assert call_kwargs["size"] == "1536x1024"
# gpt-image-2 rejects response_format — we must NOT send it.
assert "response_format" not in call_kwargs
@pytest.mark.parametrize("tier,expected_quality", [
("gpt-image-2-low", "low"),
("gpt-image-2-medium", "medium"),
("gpt-image-2-high", "high"),
])
def test_tier_maps_to_quality(self, provider, monkeypatch, tier, expected_quality):
monkeypatch.setenv("OPENAI_IMAGE_MODEL", tier)
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["model"] == tier
assert result["quality"] == expected_quality
assert fake_client.images.generate.call_args.kwargs["quality"] == expected_quality
# Always the same underlying API model regardless of tier.
assert fake_client.images.generate.call_args.kwargs["model"] == "gpt-image-2"
@pytest.mark.parametrize("aspect,expected_size", [
("landscape", "1536x1024"),
("square", "1024x1024"),
("portrait", "1024x1536"),
])
def test_aspect_ratio_mapping(self, provider, aspect, expected_size):
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
with _patched_openai(fake_client):
provider.generate("a cat", aspect_ratio=aspect)
assert fake_client.images.generate.call_args.kwargs["size"] == expected_size
def test_revised_prompt_passed_through(self, provider):
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=_b64_png(), revised_prompt="A photo of a cat",
)
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["revised_prompt"] == "A photo of a cat"
def test_api_error_returns_error_response(self, provider):
fake_client = MagicMock()
fake_client.images.generate.side_effect = RuntimeError("boom")
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "boom" in result["error"]
def test_empty_response_data(self, provider):
fake_client = MagicMock()
fake_client.images.generate.return_value = SimpleNamespace(data=[])
with _patched_openai(fake_client):
result = provider.generate("a cat")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_url_response_is_cached_locally(self, provider):
"""OpenAI URL response (if API ever returns one) is cached locally.
Pre-fix this asserted the bare URL passed through; symmetric to the
xAI #26942 fix. Even though gpt-image-2 returns b64 today, every
``image_gen`` provider must guarantee the gateway gets a stable
file path so ephemeral signed URLs can't expire mid-flight.
"""
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=None, url="https://example.com/img.png",
)
with _patched_openai(fake_client), patch(
"plugins.image_gen.openai.save_url_image",
return_value=Path("/tmp/openai_gpt-image-2_20260524_000000_deadbeef.png"),
) as mock_save_url:
result = provider.generate("a cat")
assert result["success"] is True
assert result["image"].startswith("/")
assert "example.com" not in result["image"]
mock_save_url.assert_called_once()
def test_url_response_falls_back_to_bare_url_when_download_fails(self, provider):
"""Cache failure must not turn into a tool error — symmetric with xAI."""
import requests as req_lib
fake_client = MagicMock()
fake_client.images.generate.return_value = _fake_response(
b64=None, url="https://example.com/img.png",
)
with _patched_openai(fake_client), patch(
"plugins.image_gen.openai.save_url_image",
side_effect=req_lib.HTTPError("404 from CDN"),
):
result = provider.generate("a cat")
assert result["success"] is True
assert result["image"] == "https://example.com/img.png"
@@ -0,0 +1,336 @@
#!/usr/bin/env python3
"""Tests for xAI image generation provider."""
from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _fake_api_key(monkeypatch):
"""Ensure XAI_API_KEY is set for all tests."""
monkeypatch.setenv("XAI_API_KEY", "test-key-12345")
# ---------------------------------------------------------------------------
# Provider class tests
# ---------------------------------------------------------------------------
class TestXAIImageGenProvider:
def test_name(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.name == "xai"
def test_display_name(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.display_name == "xAI (Grok)"
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("XAI_API_KEY", "sk-xxx")
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.is_available() is True
def test_is_available_without_key(self, monkeypatch):
monkeypatch.delenv("XAI_API_KEY", raising=False)
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.is_available() is False
def test_list_models(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
models = provider.list_models()
assert len(models) >= 1
assert models[0]["id"] == "grok-imagine-image"
def test_default_model(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
assert provider.default_model() == "grok-imagine-image"
def test_get_setup_schema(self):
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
schema = provider.get_setup_schema()
assert schema["name"] == "xAI Grok Imagine (image)"
assert schema["badge"] == "paid"
# Auth resolution is delegated to the shared "xai_grok" post_setup
# hook so the picker doesn't blindly prompt for XAI_API_KEY when the
# user is already signed in via xAI Grok OAuth.
assert schema["env_vars"] == []
assert schema["post_setup"] == "xai_grok"
# ---------------------------------------------------------------------------
# Config tests
# ---------------------------------------------------------------------------
class TestConfig:
def test_default_model(self):
from plugins.image_gen.xai import _resolve_model
model_id, meta = _resolve_model()
assert model_id == "grok-imagine-image"
def test_default_resolution(self):
from plugins.image_gen.xai import _resolve_resolution
assert _resolve_resolution() == "1k"
def test_custom_model(self, monkeypatch):
monkeypatch.setenv("XAI_IMAGE_MODEL", "grok-imagine-image")
from plugins.image_gen.xai import _resolve_model
model_id, _ = _resolve_model()
assert model_id == "grok-imagine-image"
# ---------------------------------------------------------------------------
# Generate tests
# ---------------------------------------------------------------------------
class TestGenerate:
def test_missing_api_key(self, monkeypatch):
monkeypatch.delenv("XAI_API_KEY", raising=False)
from plugins.image_gen.xai import XAIImageGenProvider
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert "XAI_API_KEY" in result["error"]
def test_successful_generation(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"b64_json": "dGVzdC1pbWFnZS1kYXRh"}], # base64 "test-image-data"
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
with patch("plugins.image_gen.xai.save_b64_image", return_value="/tmp/test.png"):
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True
assert result["image"] == "/tmp/test.png"
assert result["provider"] == "xai"
assert result["model"] == "grok-imagine-image"
def test_successful_url_response(self):
"""xAI URL response is cached locally — #26942 contract.
Pre-fix this asserted ``result["image"] == "<the bare URL>"``, which
was exactly the bug: xAI's ``imgen.x.ai/xai-tmp-*`` URLs expire fast
and the gateway 404'd by ``send_photo`` time. Post-fix the URL
bytes are downloaded at tool-completion and the result carries an
absolute filesystem path the gateway can upload from.
"""
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
patch(
"plugins.image_gen.xai.save_url_image",
return_value=Path("/tmp/xai_grok-imagine-image_20260524_000000_deadbeef.jpg"),
) as mock_save_url:
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True
assert result["image"].startswith("/"), (
f"URL response must be cached to an absolute path, got {result['image']!r}"
)
assert "imgen.x.ai" not in result["image"], (
"ephemeral xAI URL must not leak into result.image — caller will 404"
)
# The downloader should have been called exactly once with the URL
# and an xai-prefixed cache filename.
mock_save_url.assert_called_once()
call_args, call_kwargs = mock_save_url.call_args
assert call_args[0] == "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"
assert call_kwargs.get("prefix", "").startswith("xai_")
def test_url_response_falls_back_to_bare_url_when_download_fails(self):
"""If caching the URL fails (network blip, 404 in-flight), the
provider must NOT hard-error — fall through to returning the bare
URL so the agent surface at least sees *something*. The gateway's
existing URL-send fallback then has a chance to succeed; if it
too 404s, the user gets the original (now legible) error rather
than an opaque "image generation failed" tool result.
"""
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
patch(
"plugins.image_gen.xai.save_url_image",
side_effect=req_lib.HTTPError("404 from CDN"),
):
provider = XAIImageGenProvider()
result = provider.generate(prompt="A cat playing piano")
assert result["success"] is True, (
"Cache failure must not turn into a tool error — gateway gets a chance to retry"
)
assert result["image"] == "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"
def test_api_error(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 401
mock_resp.text = "Unauthorized"
mock_resp.json.return_value = {"error": {"message": "Invalid API key"}}
mock_resp.raise_for_status.side_effect = req_lib.HTTPError(response=mock_resp)
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
def test_api_error_preserves_real_response_status(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
response = req_lib.Response()
response.status_code = 401
response._content = json.dumps({"error": {"message": "Invalid API key"}}).encode()
response.headers["Content-Type"] = "application/json"
response.raise_for_status = MagicMock(
side_effect=req_lib.HTTPError(response=response)
)
with patch("plugins.image_gen.xai.requests.post", return_value=response):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "api_error"
assert "xAI image generation failed (401): Invalid API key" in result["error"]
def test_timeout(self):
import requests as req_lib
from plugins.image_gen.xai import XAIImageGenProvider
with patch("plugins.image_gen.xai.requests.post", side_effect=req_lib.Timeout()):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "timeout"
def test_empty_response(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {"data": []}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
provider = XAIImageGenProvider()
result = provider.generate(prompt="test")
assert result["success"] is False
assert result["error_type"] == "empty_response"
def test_auth_header(self):
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {
"data": [{"url": "https://xai.image/test.png"}],
}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
provider = XAIImageGenProvider()
provider.generate(prompt="test")
call_args = mock_post.call_args
headers = call_args.kwargs.get("headers") or call_args[1].get("headers")
assert "Bearer test-key-12345" in headers["Authorization"]
assert "Hermes-Agent" in headers["User-Agent"]
def test_payload_resolution_is_literal_1k_or_2k(self):
"""Regression: xAI API rejects numeric resolutions ("1024"/"2048") with 422.
The endpoint expects the literal strings "1k" or "2k". Ensure the wire
payload carries that literal — not a numeric mapping. See PR #18678.
"""
from plugins.image_gen.xai import XAIImageGenProvider
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
mock_resp.json.return_value = {"data": [{"url": "https://xai.image/test.png"}]}
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
provider = XAIImageGenProvider()
provider.generate(prompt="test")
payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json")
assert payload["resolution"] in {"1k", "2k"}, (
f"resolution must be the literal '1k' or '2k', got {payload['resolution']!r}"
)
# ---------------------------------------------------------------------------
# Registration test
# ---------------------------------------------------------------------------
class TestRegistration:
def test_register(self):
from plugins.image_gen.xai import XAIImageGenProvider, register
mock_ctx = MagicMock()
register(mock_ctx)
mock_ctx.register_image_gen_provider.assert_called_once()
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
assert isinstance(provider, XAIImageGenProvider)
assert provider.name == "xai"
View File
File diff suppressed because it is too large Load Diff
+241
View File
@@ -0,0 +1,241 @@
"""Tests for Mem0 API v2 compatibility — filters param and dict response unwrapping.
Salvaged from PRs #5301 (qaqcvc) and #5117 (vvvanguards).
"""
import json
import os
import stat
import pytest
from plugins.memory.mem0 import Mem0MemoryProvider
class FakeClientV2:
"""Fake Mem0 client that returns v2-style dict responses and captures call kwargs."""
def __init__(self, search_results=None, all_results=None):
self._search_results = search_results or {"results": []}
self._all_results = all_results or {"results": []}
self.captured_search = {}
self.captured_get_all = {}
self.captured_add = []
def search(self, **kwargs):
self.captured_search = kwargs
return self._search_results
def get_all(self, **kwargs):
self.captured_get_all = kwargs
return self._all_results
def add(self, messages, **kwargs):
self.captured_add.append({"messages": messages, **kwargs})
# ---------------------------------------------------------------------------
# Filter migration: bare user_id= -> filters={}
# ---------------------------------------------------------------------------
class TestMem0FiltersV2:
"""All API calls must use filters={} instead of bare user_id= kwargs."""
def _make_provider(self, monkeypatch, client):
provider = Mem0MemoryProvider()
provider.initialize("test-session")
provider._user_id = "u123"
provider._agent_id = "hermes"
monkeypatch.setattr(provider, "_get_client", lambda: client)
return provider
def test_search_uses_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.handle_tool_call("mem0_search", {"query": "hello", "top_k": 3, "rerank": False})
assert client.captured_search["query"] == "hello"
assert client.captured_search["top_k"] == 3
assert client.captured_search["rerank"] is False
assert client.captured_search["filters"] == {"user_id": "u123"}
# Must NOT have bare user_id kwarg
assert "user_id" not in {k for k in client.captured_search if k != "filters"}
def test_profile_uses_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.handle_tool_call("mem0_profile", {})
assert client.captured_get_all["filters"] == {"user_id": "u123"}
assert "user_id" not in {k for k in client.captured_get_all if k != "filters"}
def test_prefetch_uses_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.queue_prefetch("hello")
provider._prefetch_thread.join(timeout=2)
assert client.captured_search["query"] == "hello"
assert client.captured_search["filters"] == {"user_id": "u123"}
assert "user_id" not in {k for k in client.captured_search if k != "filters"}
def test_sync_turn_uses_write_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.sync_turn("user said this", "assistant replied", session_id="s1")
provider._sync_thread.join(timeout=2)
assert len(client.captured_add) == 1
call = client.captured_add[0]
assert call["user_id"] == "u123"
assert call["agent_id"] == "hermes"
def test_conclude_uses_write_filters(self, monkeypatch):
client = FakeClientV2()
provider = self._make_provider(monkeypatch, client)
provider.handle_tool_call("mem0_conclude", {"conclusion": "user likes dark mode"})
assert len(client.captured_add) == 1
call = client.captured_add[0]
assert call["user_id"] == "u123"
assert call["agent_id"] == "hermes"
assert call["infer"] is False
def test_read_filters_no_agent_id(self):
"""Read filters should use user_id only — cross-session recall across agents."""
provider = Mem0MemoryProvider()
provider._user_id = "u123"
provider._agent_id = "hermes"
assert provider._read_filters() == {"user_id": "u123"}
def test_write_filters_include_agent_id(self):
"""Write filters should include agent_id for attribution."""
provider = Mem0MemoryProvider()
provider._user_id = "u123"
provider._agent_id = "hermes"
assert provider._write_filters() == {"user_id": "u123", "agent_id": "hermes"}
# ---------------------------------------------------------------------------
# Dict response unwrapping (API v2 wraps in {"results": [...]})
# ---------------------------------------------------------------------------
class TestMem0ResponseUnwrapping:
"""API v2 returns {"results": [...]} dicts; we must extract the list."""
def _make_provider(self, monkeypatch, client):
provider = Mem0MemoryProvider()
provider.initialize("test-session")
monkeypatch.setattr(provider, "_get_client", lambda: client)
return provider
def test_profile_dict_response(self, monkeypatch):
client = FakeClientV2(all_results={"results": [{"memory": "alpha"}, {"memory": "beta"}]})
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call("mem0_profile", {}))
assert result["count"] == 2
assert "alpha" in result["result"]
assert "beta" in result["result"]
def test_profile_list_response_backward_compat(self, monkeypatch):
"""Old API returned bare lists — still works."""
client = FakeClientV2(all_results=[{"memory": "gamma"}])
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call("mem0_profile", {}))
assert result["count"] == 1
assert "gamma" in result["result"]
def test_search_dict_response(self, monkeypatch):
client = FakeClientV2(search_results={
"results": [{"memory": "foo", "score": 0.9}, {"memory": "bar", "score": 0.7}]
})
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call(
"mem0_search", {"query": "test", "top_k": 5}
))
assert result["count"] == 2
assert result["results"][0]["memory"] == "foo"
def test_search_list_response_backward_compat(self, monkeypatch):
"""Old API returned bare lists — still works."""
client = FakeClientV2(search_results=[{"memory": "baz", "score": 0.8}])
provider = self._make_provider(monkeypatch, client)
result = json.loads(provider.handle_tool_call(
"mem0_search", {"query": "test"}
))
assert result["count"] == 1
def test_unwrap_results_edge_cases(self):
"""_unwrap_results handles all shapes gracefully."""
assert Mem0MemoryProvider._unwrap_results({"results": [1, 2]}) == [1, 2]
assert Mem0MemoryProvider._unwrap_results([3, 4]) == [3, 4]
assert Mem0MemoryProvider._unwrap_results({}) == []
assert Mem0MemoryProvider._unwrap_results(None) == []
assert Mem0MemoryProvider._unwrap_results("unexpected") == []
def test_prefetch_dict_response(self, monkeypatch):
client = FakeClientV2(search_results={
"results": [{"memory": "user prefers dark mode"}]
})
provider = Mem0MemoryProvider()
provider.initialize("test-session")
monkeypatch.setattr(provider, "_get_client", lambda: client)
provider.queue_prefetch("preferences")
provider._prefetch_thread.join(timeout=2)
result = provider.prefetch("preferences")
assert "dark mode" in result
# ---------------------------------------------------------------------------
# Default preservation
# ---------------------------------------------------------------------------
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode bits not enforced on Windows")
def test_save_config_sets_owner_only_permissions(tmp_path):
"""mem0.json must be written with 0o600 so API key is not world-readable."""
provider = Mem0MemoryProvider()
provider.save_config({"api_key": "m0-test-key"}, str(tmp_path))
config_file = tmp_path / "mem0.json"
assert config_file.exists()
mode = stat.S_IMODE(config_file.stat().st_mode)
assert mode == 0o600, f"Expected 0o600 (owner-only), got {oct(mode)}"
class TestMem0Defaults:
"""Ensure we don't break existing users' defaults."""
def test_default_user_id_hermes_user(self, monkeypatch, tmp_path):
monkeypatch.setenv("MEM0_API_KEY", "test-key")
monkeypatch.delenv("MEM0_USER_ID", raising=False)
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
provider = Mem0MemoryProvider()
provider.initialize("test")
assert provider._user_id == "hermes-user"
def test_default_agent_id_hermes(self, monkeypatch, tmp_path):
monkeypatch.setenv("MEM0_API_KEY", "test-key")
monkeypatch.delenv("MEM0_AGENT_ID", raising=False)
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
provider = Mem0MemoryProvider()
provider.initialize("test")
assert provider._agent_id == "hermes"
@@ -0,0 +1,422 @@
import json
import zipfile
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from plugins.memory.openviking import OpenVikingMemoryProvider, _VikingClient
def test_tool_search_sorts_by_raw_score_across_buckets():
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"result": {
"memories": [
{"uri": "viking://memories/1", "score": 0.9003, "abstract": "memory result"},
],
"resources": [
{"uri": "viking://resources/1", "score": 0.9004, "abstract": "resource result"},
],
"skills": [
{"uri": "viking://skills/1", "score": 0.8999, "abstract": "skill result"},
],
"total": 3,
}
}
result = json.loads(provider._tool_search({"query": "ranking"}))
assert [entry["uri"] for entry in result["results"]] == [
"viking://resources/1",
"viking://memories/1",
"viking://skills/1",
]
assert [entry["score"] for entry in result["results"]] == [0.9, 0.9, 0.9]
assert result["total"] == 3
def test_tool_search_sorts_missing_raw_score_after_negative_scores():
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"result": {
"memories": [
{"uri": "viking://memories/missing", "abstract": "missing score"},
],
"resources": [
{"uri": "viking://resources/negative", "score": -0.25, "abstract": "negative score"},
],
"skills": [
{"uri": "viking://skills/positive", "score": 0.1, "abstract": "positive score"},
],
"total": 3,
}
}
result = json.loads(provider._tool_search({"query": "ranking"}))
assert [entry["uri"] for entry in result["results"]] == [
"viking://skills/positive",
"viking://memories/missing",
"viking://resources/negative",
]
assert [entry["score"] for entry in result["results"]] == [0.1, 0.0, -0.25]
assert result["total"] == 3
def test_tool_add_resource_uploads_existing_local_file(tmp_path):
sample = tmp_path / "sample.md"
sample.write_text("# Local resource\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.upload_temp_file.return_value = "upload_sample.md"
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/sample"},
}
result = json.loads(provider._tool_add_resource({
"url": str(sample),
"reason": "local test",
"wait": True,
}))
provider._client.upload_temp_file.assert_called_once_with(sample)
provider._client.post.assert_called_once_with("/api/v1/resources", {
"reason": "local test",
"wait": True,
"source_name": "sample.md",
"temp_file_id": "upload_sample.md",
})
assert result["status"] == "added"
assert result["root_uri"] == "viking://resources/sample"
def test_tool_add_resource_uploads_file_uri(tmp_path):
sample = tmp_path / "sample.md"
sample.write_text("# Local resource\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.upload_temp_file.return_value = "upload_sample.md"
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/sample"},
}
result = json.loads(provider._tool_add_resource({
"url": sample.as_uri(),
"reason": "file uri test",
}))
provider._client.upload_temp_file.assert_called_once_with(sample)
provider._client.post.assert_called_once_with("/api/v1/resources", {
"reason": "file uri test",
"source_name": "sample.md",
"temp_file_id": "upload_sample.md",
})
assert result["status"] == "added"
assert result["root_uri"] == "viking://resources/sample"
def test_tool_add_resource_uploads_existing_local_directory_and_cleans_zip(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
nested = docs / "nested"
nested.mkdir()
(nested / "api.md").write_text("# API\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
uploaded_paths = []
provider._client.upload_temp_file.side_effect = (
lambda path: uploaded_paths.append(path) or "upload_docs.zip"
)
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/docs"},
}
result = json.loads(provider._tool_add_resource({
"url": str(docs),
"reason": "directory test",
"wait": True,
}))
assert uploaded_paths
assert uploaded_paths[0].suffix == ".zip"
assert not uploaded_paths[0].exists()
provider._client.post.assert_called_once_with("/api/v1/resources", {
"reason": "directory test",
"wait": True,
"source_name": "docs",
"temp_file_id": "upload_docs.zip",
})
assert result["status"] == "added"
assert result["root_uri"] == "viking://resources/docs"
def test_tool_add_resource_directory_zip_skips_symlink_escape(tmp_path):
secret = tmp_path / "outside-secret.txt"
secret.write_text("do not upload\n", encoding="utf-8")
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
link = docs / "leak.txt"
try:
link.symlink_to(secret)
except OSError as exc:
pytest.skip(f"symlinks unavailable in test environment: {exc}")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
archive_entries = {}
def inspect_upload(path):
with zipfile.ZipFile(path) as archive:
archive_entries["names"] = archive.namelist()
archive_entries["payloads"] = {
name: archive.read(name)
for name in archive.namelist()
}
return "upload_docs.zip"
provider._client.upload_temp_file.side_effect = inspect_upload
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/docs"},
}
json.loads(provider._tool_add_resource({"url": str(docs)}))
assert archive_entries["names"] == ["guide.md"]
assert b"do not upload" not in b"".join(archive_entries["payloads"].values())
def test_tool_add_resource_cleans_local_directory_zip_when_add_fails(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
uploaded_paths = []
provider._client.upload_temp_file.side_effect = (
lambda path: uploaded_paths.append(path) or "upload_docs.zip"
)
provider._client.post.side_effect = RuntimeError("add failed")
with pytest.raises(RuntimeError, match="add failed"):
provider._tool_add_resource({"url": str(docs)})
assert uploaded_paths
assert not uploaded_paths[0].exists()
def test_tool_add_resource_cleans_local_directory_zip_when_upload_fails(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
uploaded_paths = []
def fail_upload(path):
uploaded_paths.append(path)
raise RuntimeError("upload failed")
provider._client.upload_temp_file.side_effect = fail_upload
with pytest.raises(RuntimeError, match="upload failed"):
provider._tool_add_resource({"url": str(docs)})
assert uploaded_paths
assert not uploaded_paths[0].exists()
provider._client.post.assert_not_called()
def test_tool_add_resource_rejects_missing_local_path(tmp_path):
missing = tmp_path / "missing.md"
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
result = json.loads(provider._tool_add_resource({"url": str(missing)}))
assert result["error"] == f"Local resource path does not exist: {missing}"
provider._client.upload_temp_file.assert_not_called()
provider._client.post.assert_not_called()
def test_tool_add_resource_sends_remote_url_as_path():
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/remote"},
}
provider._tool_add_resource({"url": "https://example.com/doc.md"})
provider._client.upload_temp_file.assert_not_called()
provider._client.post.assert_called_once_with("/api/v1/resources", {
"path": "https://example.com/doc.md",
})
@pytest.mark.parametrize("url", [
"git@github.com:org/repo.git",
"git@ssh.dev.azure.com:v3/org/project/repo",
"ssh://git@github.com/org/repo.git",
"git://github.com/org/repo.git",
])
def test_tool_add_resource_sends_git_remote_sources_as_path(url):
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/repo"},
}
provider._tool_add_resource({"url": url})
provider._client.upload_temp_file.assert_not_called()
provider._client.post.assert_called_once_with("/api/v1/resources", {
"path": url,
})
def test_viking_client_upload_temp_file_uses_multipart_identity_headers(tmp_path, monkeypatch):
sample = tmp_path / "sample.md"
sample.write_text("# Local resource\n", encoding="utf-8")
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="test-account",
user="test-user",
agent="test-agent",
)
captured_kwargs = {}
def capture_httpx_post(url, **kwargs):
captured_kwargs.update(kwargs)
return SimpleNamespace(
status_code=200,
text="",
json=lambda: {"status": "ok", "result": {"temp_file_id": "upload_sample.md"}},
raise_for_status=lambda: None,
)
monkeypatch.setattr(client._httpx, "post", capture_httpx_post)
assert client.upload_temp_file(sample) == "upload_sample.md"
assert "files" in captured_kwargs
assert "json" not in captured_kwargs
headers = captured_kwargs["headers"]
assert headers["X-OpenViking-Account"] == "test-account"
assert headers["X-OpenViking-User"] == "test-user"
assert headers["X-OpenViking-Agent"] == "test-agent"
assert headers["X-API-Key"] == "test-key"
assert "Content-Type" not in headers
def test_viking_client_raises_structured_server_error():
client = _VikingClient.__new__(_VikingClient)
response = SimpleNamespace(
status_code=403,
text='{"status":"error"}',
json=lambda: {
"status": "error",
"error": {
"code": "PERMISSION_DENIED",
"message": "direct host filesystem paths are not allowed",
},
},
raise_for_status=lambda: None,
)
with pytest.raises(RuntimeError, match="PERMISSION_DENIED"):
client._parse_response(response)
def test_viking_client_headers_include_bearer_when_api_key_set():
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="acct",
user="usr",
agent="hermes",
)
headers = client._headers()
assert headers["X-API-Key"] == "test-key"
assert headers["Authorization"] == "Bearer test-key"
def test_viking_client_headers_send_tenant_when_default():
# account/user set to the literal string "default". OpenViking 0.3.x
# requires X-OpenViking-Account and X-OpenViking-User for ROOT API key
# requests to tenant-scoped APIs — omitting them causes
# INVALID_ARGUMENT errors even when account="default".
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="default",
user="default",
agent="hermes",
)
headers = client._headers()
assert headers["X-OpenViking-Account"] == "default"
assert headers["X-OpenViking-User"] == "default"
assert headers["X-OpenViking-Agent"] == "hermes"
assert headers["Authorization"] == "Bearer test-key"
def test_viking_client_headers_send_tenant_when_empty_falls_back_to_default():
# Empty account/user strings fall back to "default" via the constructor.
# Headers are sent even for the default value — ROOT API keys need them.
client = _VikingClient(
"https://example.com",
api_key="",
account="",
user="",
agent="hermes",
)
headers = client._headers()
assert headers["X-OpenViking-Account"] == "default"
assert headers["X-OpenViking-User"] == "default"
assert "Authorization" not in headers
assert "X-API-Key" not in headers
def test_viking_client_headers_sent_with_real_tenant_values():
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="real-account",
user="real-user",
agent="hermes",
)
headers = client._headers()
assert headers["X-OpenViking-Account"] == "real-account"
assert headers["X-OpenViking-User"] == "real-user"
def test_viking_client_health_sends_auth_headers(monkeypatch):
client = _VikingClient(
"https://example.com",
api_key="test-key",
account="",
user="",
agent="hermes",
)
captured = {}
def capture_get(url, **kwargs):
captured["url"] = url
captured["headers"] = kwargs.get("headers") or {}
return SimpleNamespace(status_code=200)
monkeypatch.setattr(client._httpx, "get", capture_get)
assert client.health() is True
assert captured["url"] == "https://example.com/health"
assert captured["headers"]["Authorization"] == "Bearer test-key"
@@ -0,0 +1,423 @@
import json
import os
import stat
import threading
import pytest
from plugins.memory.supermemory import (
SupermemoryMemoryProvider,
_clean_text_for_capture,
_format_prefetch_context,
_load_supermemory_config,
_save_supermemory_config,
)
class FakeClient:
def __init__(self, api_key: str, timeout: float, container_tag: str, search_mode: str = "hybrid"):
self.api_key = api_key
self.timeout = timeout
self.container_tag = container_tag
self.search_mode = search_mode
self.add_calls = []
self.search_results = []
self.profile_response = {"static": [], "dynamic": [], "search_results": []}
self.ingest_calls = []
self.forgotten_ids = []
self.forget_by_query_response = {"success": True, "message": "Forgot"}
def add_memory(self, content, metadata=None, *, entity_context="",
container_tag=None, custom_id=None):
self.add_calls.append({
"content": content,
"metadata": metadata,
"entity_context": entity_context,
"container_tag": container_tag,
"custom_id": custom_id,
})
return {"id": "mem_123"}
def search_memories(self, query, *, limit=5, container_tag=None, search_mode=None):
return self.search_results
def get_profile(self, query=None, *, container_tag=None):
return self.profile_response
def forget_memory(self, memory_id, *, container_tag=None):
self.forgotten_ids.append(memory_id)
def forget_by_query(self, query, *, container_tag=None):
return self.forget_by_query_response
def ingest_conversation(self, session_id, messages):
self.ingest_calls.append({"session_id": session_id, "messages": messages})
@pytest.fixture
def provider(monkeypatch, tmp_path):
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
p = SupermemoryMemoryProvider()
p.initialize("session-1", hermes_home=str(tmp_path), platform="cli")
return p
def test_is_available_false_without_api_key(monkeypatch):
monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False)
p = SupermemoryMemoryProvider()
assert p.is_available() is False
def test_is_available_false_when_import_missing(monkeypatch):
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
import builtins
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
if name == "supermemory":
raise ImportError("missing")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", fake_import)
p = SupermemoryMemoryProvider()
assert p.is_available() is False
def test_load_and_save_config_round_trip(tmp_path):
_save_supermemory_config({"container_tag": "demo-tag", "auto_capture": False}, str(tmp_path))
cfg = _load_supermemory_config(str(tmp_path))
# container_tag is kept raw — sanitization happens in initialize() after template resolution
assert cfg["container_tag"] == "demo-tag"
assert cfg["auto_capture"] is False
assert cfg["auto_recall"] is True
def test_clean_text_for_capture_strips_injected_context():
text = "hello\n<supermemory-context>ignore me</supermemory-context>\nworld"
assert _clean_text_for_capture(text) == "hello\nworld"
def test_format_prefetch_context_deduplicates_overlap():
result = _format_prefetch_context(
static_facts=["Jordan prefers short answers"],
dynamic_facts=["Jordan prefers short answers", "Uses Hermes"],
search_results=[{"memory": "Uses Hermes", "similarity": 0.9}],
max_results=10,
)
assert result.count("Jordan prefers short answers") == 1
assert result.count("Uses Hermes") == 1
assert "<supermemory-context>" in result
def test_prefetch_includes_profile_on_first_turn(provider):
provider._client.profile_response = {
"static": ["Jordan prefers short answers"],
"dynamic": ["Current project is Supermemory provider"],
"search_results": [{"memory": "Working on Hermes memory provider", "similarity": 0.88}],
}
provider.on_turn_start(1, "start")
result = provider.prefetch("what am I working on?")
assert "User Profile (Persistent)" in result
assert "Recent Context" in result
assert "Relevant Memories" in result
def test_prefetch_skips_profile_between_frequency(provider):
provider._client.profile_response = {
"static": ["Jordan prefers short answers"],
"dynamic": ["Current project is Supermemory provider"],
"search_results": [{"memory": "Working on Hermes memory provider", "similarity": 0.88}],
}
provider.on_turn_start(2, "next")
result = provider.prefetch("what am I working on?")
assert "Relevant Memories" in result
assert "User Profile (Persistent)" not in result
def test_sync_turn_skips_trivial_message(provider):
provider.sync_turn("ok", "sure", session_id="session-1")
assert provider._client.add_calls == []
def test_sync_turn_persists_cleaned_exchange(provider):
provider.sync_turn(
"Please remember this\n<supermemory-context>ignore</supermemory-context>",
"Got it, storing the context",
session_id="session-1",
)
provider._sync_thread.join(timeout=1)
assert len(provider._client.add_calls) == 1
content = provider._client.add_calls[0]["content"]
assert "ignore" not in content
assert "[role: user]" in content
assert "[role: assistant]" in content
def test_on_session_end_ingests_clean_messages(provider):
messages = [
{"role": "system", "content": "skip"},
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi there"},
]
provider.on_session_end(messages)
assert len(provider._client.ingest_calls) == 1
payload = provider._client.ingest_calls[0]
assert payload["session_id"] == "session-1"
assert payload["messages"] == [
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi there"},
]
def test_on_memory_write_tracks_thread(provider):
provider.on_memory_write("add", "memory", "Jordan likes concise docs")
assert provider._write_thread is not None
provider._write_thread.join(timeout=1)
assert len(provider._client.add_calls) == 1
assert provider._client.add_calls[0]["metadata"]["type"] == "explicit_memory"
def test_shutdown_joins_and_clears_threads(provider, monkeypatch):
started = threading.Event()
release = threading.Event()
def slow_add_memory(content, metadata=None, *, entity_context="",
container_tag=None, custom_id=None):
started.set()
release.wait(timeout=1)
provider._client.add_calls.append({
"content": content,
"metadata": metadata,
"entity_context": entity_context,
})
return {"id": "mem_slow"}
monkeypatch.setattr(provider._client, "add_memory", slow_add_memory)
provider.sync_turn(
"Please remember this request in long-term memory",
"Absolutely, I will keep that in long-term memory.",
session_id="session-1",
)
assert started.wait(timeout=1)
assert provider._sync_thread is not None
started.clear()
provider.on_memory_write("add", "memory", "Jordan likes concise docs")
assert started.wait(timeout=1)
assert provider._write_thread is not None
release.set()
provider.shutdown()
assert provider._sync_thread is None
assert provider._write_thread is None
assert provider._prefetch_thread is None
assert len(provider._client.add_calls) == 2
def test_store_tool_returns_saved_payload(provider):
result = json.loads(provider.handle_tool_call("supermemory_store", {"content": "Jordan likes concise docs"}))
assert result["saved"] is True
assert result["id"] == "mem_123"
def test_search_tool_formats_results(provider):
provider._client.search_results = [
{"id": "m1", "memory": "Jordan likes concise docs", "similarity": 0.92}
]
result = json.loads(provider.handle_tool_call("supermemory_search", {"query": "concise docs"}))
assert result["count"] == 1
assert result["results"][0]["similarity"] == 92
def test_forget_tool_by_id(provider):
result = json.loads(provider.handle_tool_call("supermemory_forget", {"id": "m1"}))
assert result == {"forgotten": True, "id": "m1"}
assert provider._client.forgotten_ids == ["m1"]
def test_forget_tool_by_query(provider):
provider._client.forget_by_query_response = {"success": True, "message": "Forgot one", "id": "m7"}
result = json.loads(provider.handle_tool_call("supermemory_forget", {"query": "that thing"}))
assert result["success"] is True
assert result["id"] == "m7"
def test_profile_tool_formats_sections(provider):
provider._client.profile_response = {
"static": ["Jordan prefers concise docs"],
"dynamic": ["Working on Supermemory provider"],
"search_results": [],
}
result = json.loads(provider.handle_tool_call("supermemory_profile", {}))
assert result["static_count"] == 1
assert result["dynamic_count"] == 1
assert "User Profile (Persistent)" in result["profile"]
def test_handle_tool_call_returns_error_when_unconfigured(monkeypatch):
monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False)
p = SupermemoryMemoryProvider()
result = json.loads(p.handle_tool_call("supermemory_search", {"query": "x"}))
assert "error" in result
# -- Identity template tests --------------------------------------------------
def test_identity_template_resolved_in_container_tag(monkeypatch, tmp_path):
"""container_tag with {identity} resolves to profile-scoped tag."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"container_tag": "hermes-{identity}"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli", agent_identity="coder")
assert p._container_tag == "hermes_coder"
def test_identity_template_default_profile(monkeypatch, tmp_path):
"""Without agent_identity kwarg, {identity} resolves to 'default'."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"container_tag": "hermes-{identity}"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._container_tag == "hermes_default"
def test_container_tag_env_var_override(monkeypatch, tmp_path):
"""SUPERMEMORY_CONTAINER_TAG env var overrides config."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setenv("SUPERMEMORY_CONTAINER_TAG", "env-override")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._container_tag == "env_override"
# -- Search mode tests --------------------------------------------------------
def test_search_mode_config_passed_to_client(monkeypatch, tmp_path):
"""search_mode from config is passed to _SupermemoryClient."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"search_mode": "memories"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._search_mode == "memories"
assert p._client.search_mode == "memories"
def test_invalid_search_mode_falls_back_to_default(monkeypatch, tmp_path):
"""Invalid search_mode falls back to 'hybrid'."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({"search_mode": "invalid_mode"}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._search_mode == "hybrid"
# -- Multi-container tests ----------------------------------------------------
def test_multi_container_disabled_by_default(provider):
"""Multi-container is off by default; schemas have no container_tag param."""
assert provider._enable_custom_containers is False
schemas = provider.get_tool_schemas()
for s in schemas:
assert "container_tag" not in s["parameters"]["properties"]
def test_multi_container_enabled_adds_schema_param(monkeypatch, tmp_path):
"""When enabled, tool schemas include container_tag parameter."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["project-alpha", "shared"],
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
assert p._enable_custom_containers is True
assert p._allowed_containers == ["hermes", "project_alpha", "shared"]
schemas = p.get_tool_schemas()
for s in schemas:
assert "container_tag" in s["parameters"]["properties"]
def test_multi_container_tool_store_with_custom_tag(monkeypatch, tmp_path):
"""supermemory_store uses the resolved container_tag when multi-container is enabled."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["project-alpha"],
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
result = json.loads(p.handle_tool_call("supermemory_store", {
"content": "test memory",
"container_tag": "project-alpha",
}))
assert result["saved"] is True
assert result["container_tag"] == "project_alpha"
assert p._client.add_calls[-1]["container_tag"] == "project_alpha"
def test_multi_container_rejects_unlisted_tag(monkeypatch, tmp_path):
"""Tool calls with a non-whitelisted container_tag return an error."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["allowed-tag"],
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
result = json.loads(p.handle_tool_call("supermemory_store", {
"content": "test",
"container_tag": "forbidden-tag",
}))
assert "error" in result
assert "not allowed" in result["error"]
def test_multi_container_system_prompt_includes_instructions(monkeypatch, tmp_path):
"""system_prompt_block includes container list and instructions when multi-container is enabled."""
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
_save_supermemory_config({
"enable_custom_container_tags": True,
"custom_containers": ["docs"],
"custom_container_instructions": "Use docs for documentation context.",
}, str(tmp_path))
p = SupermemoryMemoryProvider()
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
block = p.system_prompt_block()
assert "Multi-container mode enabled" in block
assert "docs" in block
assert "Use docs for documentation context." in block
def test_get_config_schema_minimal():
"""get_config_schema only returns the API key field."""
p = SupermemoryMemoryProvider()
schema = p.get_config_schema()
assert len(schema) == 1
assert schema[0]["key"] == "api_key"
assert schema[0]["secret"] is True
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode bits not enforced on Windows")
def test_save_config_sets_owner_only_permissions(tmp_path):
"""supermemory.json must be written with 0o600 so API key is not world-readable."""
_save_supermemory_config({"api_key": "sm-test-key"}, str(tmp_path))
config_file = tmp_path / "supermemory.json"
assert config_file.exists()
mode = stat.S_IMODE(config_file.stat().st_mode)
assert mode == 0o600, f"Expected 0o600 (owner-only), got {oct(mode)}"
@@ -0,0 +1,207 @@
"""Unit tests for the DeepSeek provider profile's thinking-mode wiring.
DeepSeek V4 (and the legacy ``deepseek-reasoner``) expects every request to
carry an explicit ``extra_body.thinking`` parameter. Omitting it makes the
server default to thinking-mode ON, which then enforces the
``reasoning_content``-must-be-echoed-back contract on subsequent turns and
breaks the conversation with HTTP 400 (#15700, #17212, #17825).
These tests pin the profile's wire-shape contract so DeepSeek requests stay
correctly shaped without going live.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def deepseek_profile():
"""Resolve the registered DeepSeek profile.
Going through ``providers.get_provider_profile`` keeps the test honest —
if someone later replaces the registered class with a plain
``ProviderProfile``, every assertion below collapses.
"""
# ``model_tools`` triggers plugin discovery on import, which is what
# registers the DeepSeek profile in the global provider registry.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("deepseek")
assert profile is not None, "deepseek provider profile must be registered"
return profile
class TestDeepSeekThinkingWireShape:
"""``build_api_kwargs_extras`` produces DeepSeek's exact wire format."""
def test_v4_pro_default_enables_thinking_without_effort(self, deepseek_profile):
"""No reasoning_config → thinking enabled, server picks default effort."""
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config=None, model="deepseek-v4-pro"
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_v4_pro_enabled_with_high_effort(self, deepseek_profile):
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_standard_efforts_pass_through(self, deepseek_profile, effort):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek-v4-pro",
)
assert top_level == {"reasoning_effort": effort}
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
def test_xhigh_and_max_normalize_to_max(self, deepseek_profile, effort):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek-v4-pro",
)
assert top_level == {"reasoning_effort": "max"}
def test_explicitly_disabled_sends_disabled_marker(self, deepseek_profile):
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
The crucial bit is that the parameter is *sent* at all — DeepSeek
defaults to thinking-on when ``thinking`` is absent.
"""
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}, model="deepseek-v4-pro"
)
assert extra_body == {"thinking": {"type": "disabled"}}
# No effort when disabled — DeepSeek rejects it.
assert top_level == {}
def test_disabled_ignores_effort_field(self, deepseek_profile):
"""Effort silently dropped when thinking is off."""
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"},
model="deepseek-v4-pro",
)
assert top_level == {}
def test_unknown_effort_omits_top_level(self, deepseek_profile):
"""Garbage effort → omit reasoning_effort so DeepSeek applies its default."""
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "garbage"},
model="deepseek-v4-pro",
)
assert top_level == {}
def test_empty_effort_omits_top_level(self, deepseek_profile):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": ""},
model="deepseek-v4-pro",
)
assert top_level == {}
class TestDeepSeekModelGating:
"""V4 family + ``deepseek-reasoner`` get thinking; V3 stays untouched."""
@pytest.mark.parametrize(
"model",
[
"deepseek-v4-pro",
"deepseek-v4-flash",
"deepseek-v4-future-variant",
"deepseek-reasoner",
"DEEPSEEK-V4-PRO", # case-insensitive
],
)
def test_thinking_capable_models_emit_thinking(self, deepseek_profile, model):
extra_body, _ = deepseek_profile.build_api_kwargs_extras(
reasoning_config=None, model=model
)
assert extra_body == {"thinking": {"type": "enabled"}}
@pytest.mark.parametrize(
"model",
[
"deepseek-chat", # V3 alias
"deepseek-v3-0324", # explicit V3
"deepseek-v3.1", # V3 minor revisions
"", # bare/unknown
None, # missing
"deepseek-unknown", # unrecognized
],
)
def test_non_thinking_models_emit_nothing(self, deepseek_profile, model):
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"}, model=model
)
assert extra_body == {}
assert top_level == {}
class TestDeepSeekFullKwargsIntegration:
"""End-to-end: the transport's full kwargs match DeepSeek's live wire format.
The live test harness in ``tests/run_agent/test_deepseek_v4_thinking_live.py``
sends ``{"reasoning_effort": "high", "extra_body": {"thinking": {"type":
"enabled"}}}``. Confirm the transport produces that exact shape when wired
through the registered DeepSeek profile.
"""
def test_full_kwargs_match_live_wire_shape(self, deepseek_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=deepseek_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://api.deepseek.com/v1",
provider_name="deepseek",
)
assert kwargs["model"] == "deepseek-v4-pro"
assert kwargs["reasoning_effort"] == "high"
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
def test_v3_chat_full_kwargs_omit_thinking(self, deepseek_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-chat",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=deepseek_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://api.deepseek.com/v1",
provider_name="deepseek",
)
assert "reasoning_effort" not in kwargs
assert "extra_body" not in kwargs or "thinking" not in kwargs.get("extra_body", {})
class TestDeepSeekAuxModel:
"""DeepSeek aux model is set on the profile so users stop seeing the
bogus 'No auxiliary LLM provider configured' warning (#26924).
Pinned at the profile layer rather than the legacy
`_API_KEY_PROVIDER_AUX_MODELS_FALLBACK` dict — new providers are
expected to set `default_aux_model` on `ProviderProfile`, and the
fallback dict only exists for providers that predate the profiles
system.
"""
def test_profile_advertises_deepseek_chat(self, deepseek_profile):
assert deepseek_profile.default_aux_model == "deepseek-chat"
def test_consumer_api_returns_deepseek_chat(self):
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("deepseek") == "deepseek-chat"
def test_consumer_api_returns_non_empty(self):
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("deepseek") != ""
@@ -0,0 +1,180 @@
"""Unit tests for OpenCode Go reasoning-control wiring."""
from __future__ import annotations
import pytest
@pytest.fixture
def opencode_go_profile():
"""Resolve the registered OpenCode Go provider profile."""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("opencode-go")
assert profile is not None, "opencode-go provider profile must be registered"
return profile
class TestOpenCodeGoKimiReasoning:
"""Kimi K2 models use Moonshot's thinking + reasoning_effort shape on OpenCode Go."""
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
model="kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
def test_minimal_effort_enables_thinking_without_effort(self, opencode_go_profile):
# "minimal" is not a Moonshot-supported value — drop it, keep thinking on.
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "minimal"},
model="kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize(
"effort",
[
"xhigh",
"max",
],
)
def test_strong_efforts_clamp_to_high(self, opencode_go_profile, effort):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="moonshotai/kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
def test_low_and_medium_pass_through(self, opencode_go_profile):
for effort in ("low", "medium"):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="kimi-k2.5",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": effort}
def test_no_config_preserves_server_default(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config=None,
model="kimi-k2.6",
)
assert extra_body == {}
assert top_level == {}
class TestOpenCodeGoDeepSeekThinking:
"""DeepSeek V4 models use DeepSeek-style thinking controls on OpenCode Go."""
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
def test_no_config_emits_thinking_enabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config=None,
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_minimal_effort_enables_thinking_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "minimal"},
model="deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_xhigh_and_max_normalize_to_max(self, opencode_go_profile):
for effort in ("xhigh", "max"):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek/deepseek-v4-pro",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "max"}
class TestOpenCodeGoModelGating:
"""Other OpenCode Go models must not receive Kimi/DeepSeek controls."""
@pytest.mark.parametrize(
"model",
[
"glm-5.1",
"qwen3.6-plus",
"minimax-m2.7",
"deepseek-v3.1",
"deepseek-chat",
"",
None,
],
)
def test_non_target_models_emit_nothing(self, opencode_go_profile, model):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model=model,
)
assert extra_body == {}
assert top_level == {}
class TestOpenCodeGoFullKwargsIntegration:
"""End-to-end transport kwargs include the profile-provided controls."""
def test_kimi_reasoning_reaches_extra_body_and_top_level(self, opencode_go_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="kimi-k2.6",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_go_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://opencode.ai/zen/go/v1",
)
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
assert kwargs["reasoning_effort"] == "high"
def test_deepseek_thinking_reaches_extra_body_and_top_level(
self, opencode_go_profile
):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_go_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://opencode.ai/zen/go/v1",
)
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
assert kwargs["reasoning_effort"] == "high"
+378
View File
@@ -0,0 +1,378 @@
"""Tests for the bundled hermes-achievements dashboard plugin.
These target the two behaviors that matter for official integration:
* The 200-session scan cap is removed — the plugin now walks the entire
session history by default. Lifetime badges (tens of thousands of
tool calls) were unreachable before this fix on long-running installs.
* First-ever scans run in a background thread so the dashboard request
path never blocks, even on 8000+ session databases where a cold scan
takes minutes.
The upstream repo ships its own unittest suite under
``plugins/hermes-achievements/tests/`` covering the achievement engine
internals (tier math, secret-state handling, catalog invariants). These
tests live at the hermes-agent level and focus on the integration
contract: the plugin scans ALL of your sessions, not the first 200.
"""
from __future__ import annotations
import importlib.util
import sys
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
import pytest
PLUGIN_MODULE_PATH = (
Path(__file__).resolve().parents[2]
/ "plugins"
/ "hermes-achievements"
/ "dashboard"
/ "plugin_api.py"
)
@pytest.fixture
def plugin_api(tmp_path, monkeypatch):
"""Load plugin_api with isolated ~/.hermes so state/snapshot files don't collide.
We load the module fresh per test because the plugin keeps module-level
caches (``_SNAPSHOT_CACHE``, ``_SCAN_STATUS``, background thread handle).
Reloading gives each test a clean world.
"""
monkeypatch.setattr(Path, "home", lambda: tmp_path)
spec = importlib.util.spec_from_file_location(
f"plugin_api_test_{id(tmp_path)}", PLUGIN_MODULE_PATH
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# Stash monkeypatch so ``_install_fake_session_db`` can use it to
# swap ``sys.modules['hermes_state']`` with auto-restoration. Without
# this, a raw ``sys.modules[...] = fake`` assignment would leak the
# fake into later tests in the same xdist worker — breaking every
# test that does ``from hermes_state import SessionDB``.
module._test_monkeypatch = monkeypatch
yield module
class _FakeSessionDB:
"""Stand-in for hermes_state.SessionDB that records scan calls."""
def __init__(self, session_count: int, scan_delay: float = 0):
self.session_count = session_count
self.scan_delay = scan_delay
self.last_limit: Optional[int] = None
self.last_include_children: Optional[bool] = None
self.list_calls = 0
self.messages_calls = 0
def list_sessions_rich(
self,
source: Optional[str] = None,
exclude_sources: Optional[List[str]] = None,
limit: int = 20,
offset: int = 0,
include_children: bool = False,
project_compression_tips: bool = True,
) -> List[Dict[str, Any]]:
if self.scan_delay:
time.sleep(self.scan_delay)
self.last_limit = limit
self.last_include_children = include_children
self.list_calls += 1
# SQLite semantics: LIMIT -1 = unlimited. Honor that here.
effective = self.session_count if limit == -1 else min(self.session_count, limit)
now = int(time.time())
return [
{
"id": f"sess-{i}",
"title": f"Session {i}",
"preview": f"preview {i}",
"started_at": now - (self.session_count - i) * 60,
"last_active": now - (self.session_count - i) * 60 + 30,
"source": "cli",
"model": "test-model",
}
for i in range(effective)
]
def get_messages(self, session_id: str) -> List[Dict[str, Any]]:
self.messages_calls += 1
return [
{"role": "user", "content": f"ask {session_id}"},
{
"role": "assistant",
"tool_calls": [{"function": {"name": "terminal"}}],
},
{"role": "tool", "tool_name": "terminal", "content": "ok"},
]
def close(self) -> None:
pass
def _install_fake_session_db(plugin_api, fake_db):
"""Inject a fake SessionDB so ``scan_sessions`` finds it via its local import.
Uses the monkeypatch stashed on ``plugin_api`` by the fixture, so the
``sys.modules['hermes_state']`` swap is auto-restored at test teardown
and cannot leak into unrelated tests in the same xdist worker.
"""
fake_module = type(sys)("hermes_state")
fake_module.SessionDB = lambda: fake_db
plugin_api._test_monkeypatch.setitem(sys.modules, "hermes_state", fake_module)
def test_scan_sessions_default_scans_all_history_not_first_200(plugin_api):
"""Bug regression: ``scan_sessions()`` used to cap at limit=200.
A user with 8000+ sessions would only see ~2% of their history in
achievement totals, making lifetime badges unreachable. The default
now passes ``LIMIT -1`` (SQLite "unlimited") to ``list_sessions_rich``.
"""
fake_db = _FakeSessionDB(session_count=500) # > old 200 cap
_install_fake_session_db(plugin_api, fake_db)
result = plugin_api.scan_sessions()
assert fake_db.last_limit == -1, (
"scan_sessions() must pass LIMIT=-1 (unlimited) to list_sessions_rich "
f"by default, got {fake_db.last_limit}"
)
assert fake_db.last_include_children is True, (
"scan_sessions() must include subagent/compression child sessions so "
"tool calls made in delegated agents still count toward achievements"
)
assert len(result["sessions"]) == 500
assert result["scan_meta"]["sessions_total"] == 500
def test_scan_sessions_explicit_positive_limit_is_honored(plugin_api):
"""Callers can still pass a small limit for smoke tests."""
fake_db = _FakeSessionDB(session_count=500)
_install_fake_session_db(plugin_api, fake_db)
result = plugin_api.scan_sessions(limit=10)
assert fake_db.last_limit == 10
assert len(result["sessions"]) == 10
def test_scan_sessions_zero_or_negative_limit_means_unlimited(plugin_api):
"""``limit=0`` and ``limit=-1`` both map to the unlimited path."""
fake_db = _FakeSessionDB(session_count=300)
_install_fake_session_db(plugin_api, fake_db)
plugin_api.scan_sessions(limit=0)
assert fake_db.last_limit == -1
plugin_api.scan_sessions(limit=-1)
assert fake_db.last_limit == -1
def test_evaluate_all_first_run_returns_pending_and_starts_background_scan(plugin_api):
"""First-ever evaluate_all with no cache returns a pending placeholder
immediately and kicks off a background scan thread. Cold scans on
large DBs take minutes — blocking the dashboard request path is not
acceptable.
"""
fake_db = _FakeSessionDB(session_count=50)
_install_fake_session_db(plugin_api, fake_db)
# Wrap _run_scan_and_update_cache so we can release it on demand,
# simulating a slow cold scan without actually waiting.
scan_started = threading.Event()
allow_scan_finish = threading.Event()
original_run = plugin_api._run_scan_and_update_cache
def gated_run(*args, **kwargs):
scan_started.set()
allow_scan_finish.wait(timeout=5)
original_run(*args, **kwargs)
plugin_api._run_scan_and_update_cache = gated_run
t0 = time.time()
result = plugin_api.evaluate_all()
elapsed = time.time() - t0
# Immediate return — should not block waiting for the scan.
assert elapsed < 1.0, f"evaluate_all blocked for {elapsed:.2f}s on first run"
assert result["scan_meta"]["mode"] == "pending"
assert result["unlocked_count"] == 0
# Catalog still rendered so UI has something to draw.
assert result["total_count"] >= 60
# Background scan is running.
assert scan_started.wait(timeout=2), "background scan did not start"
# Let the scan complete, then a second call returns real data.
allow_scan_finish.set()
# Wait for thread to finish.
thread = plugin_api._BACKGROUND_SCAN_THREAD
assert thread is not None
thread.join(timeout=5)
assert not thread.is_alive()
second = plugin_api.evaluate_all()
assert second["scan_meta"]["mode"] != "pending"
assert second["scan_meta"].get("sessions_total") == 50
def test_evaluate_all_stale_cache_serves_stale_and_refreshes_in_background(plugin_api):
"""When the snapshot is on-disk but older than TTL, evaluate_all returns
the stale data immediately and kicks a background refresh. Users don't
stare at a loading spinner every time TTL expires.
"""
fake_db = _FakeSessionDB(session_count=10, scan_delay=2.0)
_install_fake_session_db(plugin_api, fake_db)
stale_generated_at = int(time.time()) - plugin_api.SNAPSHOT_TTL_SECONDS - 60
stale_payload = {
"achievements": [],
"sessions": [],
"aggregate": {},
"scan_meta": {"mode": "full", "sessions_total": 1, "sessions_rescanned": 1, "sessions_reused": 0},
"error": None,
"unlocked_count": 0,
"discovered_count": 0,
"secret_count": 0,
"total_count": 0,
"generated_at": stale_generated_at,
}
plugin_api.save_snapshot(stale_payload)
t0 = time.time()
result = plugin_api.evaluate_all()
elapsed = time.time() - t0
assert elapsed < 1.0, f"evaluate_all blocked for {elapsed:.2f}s serving stale data"
assert result["generated_at"] == stale_generated_at
# Background scan should be running or have completed.
thread = plugin_api._BACKGROUND_SCAN_THREAD
assert thread is not None
thread.join(timeout=5)
fresh = plugin_api.evaluate_all()
assert fresh["generated_at"] >= stale_generated_at
def test_evaluate_all_force_runs_synchronously(plugin_api):
"""Manual /rescan (force=True) blocks the caller — users clicking
the rescan button expect up-to-date data when the call returns.
"""
fake_db = _FakeSessionDB(session_count=25)
_install_fake_session_db(plugin_api, fake_db)
result = plugin_api.evaluate_all(force=True)
# Synchronous — snapshot is fresh on return.
assert result["scan_meta"].get("sessions_total") == 25
assert result["scan_meta"]["mode"] in {"full", "incremental"}
def test_start_background_scan_is_idempotent_while_running(plugin_api):
"""Multiple concurrent dashboard requests must not spawn duplicate scans."""
fake_db = _FakeSessionDB(session_count=5)
_install_fake_session_db(plugin_api, fake_db)
release = threading.Event()
original_run = plugin_api._run_scan_and_update_cache
def gated_run(*args, **kwargs):
release.wait(timeout=5)
original_run(*args, **kwargs)
plugin_api._run_scan_and_update_cache = gated_run
plugin_api._start_background_scan()
first_thread = plugin_api._BACKGROUND_SCAN_THREAD
assert first_thread is not None and first_thread.is_alive()
plugin_api._start_background_scan()
plugin_api._start_background_scan()
assert plugin_api._BACKGROUND_SCAN_THREAD is first_thread
release.set()
first_thread.join(timeout=5)
def test_background_scan_publishes_partial_snapshots(plugin_api):
"""The background scanner publishes intermediate snapshots to the cache
every ~N sessions. Each dashboard refresh during a long cold scan sees
more badges unlocked instead of staring at zeros for minutes and then
having everything pop at the end.
"""
fake_db = _FakeSessionDB(session_count=750)
_install_fake_session_db(plugin_api, fake_db)
# Record every partial snapshot the scanner publishes.
partial_snapshots: List[Dict[str, Any]] = []
original_compute_from_scan = plugin_api._compute_from_scan
def recording_compute(scan, *, is_partial=False):
result = original_compute_from_scan(scan, is_partial=is_partial)
if is_partial:
partial_snapshots.append(result)
return result
plugin_api._compute_from_scan = recording_compute
# scan 750 sessions with progress_every=250 → expect 2 intermediate
# publications (at 250 and 500; the final 750 call goes through the
# finished, non-partial path).
plugin_api._run_scan_and_update_cache(publish_partial_snapshots=True)
assert len(partial_snapshots) >= 2, (
f"expected at least 2 partial publications on a 750-session scan with "
f"progress_every=250, got {len(partial_snapshots)}"
)
# Partial snapshots should report growing session counts.
counts = [p["scan_meta"].get("sessions_scanned_so_far") for p in partial_snapshots]
assert counts == sorted(counts), f"partial session counts not monotonic: {counts}"
assert counts[0] < 750 and counts[-1] < 750, (
f"partial counts should be less than the final total; got {counts}"
)
# Every partial reports the expected end-state total so the UI can
# show an accurate progress bar.
for p in partial_snapshots:
assert p["scan_meta"].get("sessions_expected_total") == 750
# Final snapshot in cache is the real (non-partial) one.
final = plugin_api._SNAPSHOT_CACHE
assert final is not None
assert final["scan_meta"].get("mode") != "in_progress"
assert final["scan_meta"].get("sessions_total") == 750
def test_partial_snapshots_do_not_persist_unlock_timestamps(plugin_api):
"""Intermediate snapshots must not write to state.json — an unlock
that appears at 30% scan progress could disappear when a later session
rebalances the aggregate. Only the final snapshot records ``unlocked_at``.
"""
fake_db = _FakeSessionDB(session_count=10)
_install_fake_session_db(plugin_api, fake_db)
# Seed empty state, then invoke partial compute directly.
plugin_api.save_state({"unlocks": {}})
partial_scan = {
"sessions": [{"session_id": "x", "tool_call_count": 99999, "tool_names": set()}],
"aggregate": {"max_tool_calls_in_session": 99999, "total_tool_calls": 99999},
"scan_meta": {"mode": "in_progress"},
}
result = plugin_api._compute_from_scan(partial_scan, is_partial=True)
# Some achievements should evaluate as unlocked in this aggregate...
assert any(a["unlocked"] for a in result["achievements"])
# ...but state.json on disk stays empty (no timestamps were recorded).
persisted = plugin_api.load_state()
assert persisted.get("unlocks", {}) == {}, (
"partial scans must not record unlock timestamps — a later session "
"could change whether the badge deserves to be unlocked yet"
)
+455
View File
@@ -0,0 +1,455 @@
"""Tests for the disk-cleanup plugin.
Covers the bundled plugin at ``plugins/disk-cleanup/``:
* ``disk_cleanup`` library: track / forget / dry_run / quick / status,
``is_safe_path`` and ``guess_category`` filtering.
* Plugin ``__init__``: ``post_tool_call`` hook auto-tracks files created
by ``write_file`` / ``terminal``; ``on_session_end`` hook runs quick
cleanup when anything was tracked during the turn.
* Slash command handler: status / dry-run / quick / track / forget /
unknown subcommand behaviours.
* Bundled-plugin discovery via ``PluginManager.discover_and_load``.
"""
import importlib
import json
import sys
from pathlib import Path
import pytest
@pytest.fixture(autouse=True)
def _isolate_env(tmp_path, monkeypatch):
"""Isolate HERMES_HOME for each test.
The global hermetic fixture already redirects HERMES_HOME to a tempdir,
but we want the plugin to work with a predictable subpath. We reset
HERMES_HOME here for clarity.
"""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
def _load_lib():
"""Import the plugin's library module directly from the repo path."""
repo_root = Path(__file__).resolve().parents[2]
lib_path = repo_root / "plugins" / "disk-cleanup" / "disk_cleanup.py"
spec = importlib.util.spec_from_file_location(
"disk_cleanup_under_test", lib_path
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _load_plugin_init():
"""Import the plugin's __init__.py (which depends on the library)."""
repo_root = Path(__file__).resolve().parents[2]
plugin_dir = repo_root / "plugins" / "disk-cleanup"
# Use the PluginManager's module naming convention so relative imports work.
spec = importlib.util.spec_from_file_location(
"hermes_plugins.disk_cleanup",
plugin_dir / "__init__.py",
submodule_search_locations=[str(plugin_dir)],
)
# Ensure parent namespace package exists for the relative `. import disk_cleanup`
import types
if "hermes_plugins" not in sys.modules:
ns = types.ModuleType("hermes_plugins")
ns.__path__ = []
sys.modules["hermes_plugins"] = ns
mod = importlib.util.module_from_spec(spec)
mod.__package__ = "hermes_plugins.disk_cleanup"
mod.__path__ = [str(plugin_dir)]
sys.modules["hermes_plugins.disk_cleanup"] = mod
spec.loader.exec_module(mod)
return mod
# ---------------------------------------------------------------------------
# Library tests
# ---------------------------------------------------------------------------
class TestIsSafePath:
def test_accepts_path_under_hermes_home(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "subdir" / "file.txt"
p.parent.mkdir()
p.write_text("x")
assert dg.is_safe_path(p) is True
def test_rejects_outside_hermes_home(self, _isolate_env):
dg = _load_lib()
assert dg.is_safe_path(Path("/etc/passwd")) is False
def test_accepts_tmp_hermes_prefix(self, _isolate_env, tmp_path):
dg = _load_lib()
assert dg.is_safe_path(Path("/tmp/hermes-abc/x.log")) is True
def test_rejects_plain_tmp(self, _isolate_env):
dg = _load_lib()
assert dg.is_safe_path(Path("/tmp/other.log")) is False
def test_rejects_windows_mount(self, _isolate_env):
dg = _load_lib()
assert dg.is_safe_path(Path("/mnt/c/Users/x/test.txt")) is False
class TestGuessCategory:
def test_test_prefix(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "test_foo.py"
p.write_text("x")
assert dg.guess_category(p) == "test"
def test_tmp_prefix(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "tmp_foo.log"
p.write_text("x")
assert dg.guess_category(p) == "test"
def test_dot_test_suffix(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "mything.test.js"
p.write_text("x")
assert dg.guess_category(p) == "test"
def test_skips_protected_top_level(self, _isolate_env):
dg = _load_lib()
logs_dir = _isolate_env / "logs"
logs_dir.mkdir()
p = logs_dir / "test_log.txt"
p.write_text("x")
# Even though it matches test_* pattern, logs/ is excluded.
assert dg.guess_category(p) is None
def test_cron_subtree_categorised(self, _isolate_env):
dg = _load_lib()
# Only files under ``cron/output/`` are disposable run artifacts.
output_dir = _isolate_env / "cron" / "output" / "job_123"
output_dir.mkdir(parents=True)
p = output_dir / "run.md"
p.write_text("x")
assert dg.guess_category(p) == "cron-output"
def test_cron_jobs_json_not_tracked(self, _isolate_env):
"""Regression for #32164: the cron registry must never be tracked."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
p = cron_dir / "jobs.json"
p.write_text("[]")
assert dg.guess_category(p) is None
def test_cron_tick_lock_not_tracked(self, _isolate_env):
"""Regression for #32164: cron tick-lock is control-plane state."""
dg = _load_lib()
cron_dir = _isolate_env / "cron"
cron_dir.mkdir()
p = cron_dir / ".tick.lock"
p.write_text("")
assert dg.guess_category(p) is None
def test_cronjobs_top_level_not_tracked(self, _isolate_env):
"""The legacy ``cronjobs`` alias is also control-plane at the top."""
dg = _load_lib()
cron_dir = _isolate_env / "cronjobs"
cron_dir.mkdir()
p = cron_dir / "jobs.json"
p.write_text("[]")
assert dg.guess_category(p) is None
def test_ordinary_file_returns_none(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "notes.md"
p.write_text("x")
assert dg.guess_category(p) is None
class TestTrackForgetQuick:
def test_track_then_quick_deletes_test(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "test_a.py"
p.write_text("x")
assert dg.track(str(p), "test", silent=True) is True
summary = dg.quick()
assert summary["deleted"] == 1
assert not p.exists()
def test_track_dedup(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "test_a.py"
p.write_text("x")
assert dg.track(str(p), "test", silent=True) is True
# Second call returns False (already tracked)
assert dg.track(str(p), "test", silent=True) is False
def test_track_rejects_outside_home(self, _isolate_env):
dg = _load_lib()
# /etc/hostname exists on most Linux boxes; fall back if not.
outside = "/etc/hostname" if Path("/etc/hostname").exists() else "/etc/passwd"
assert dg.track(outside, "test", silent=True) is False
def test_track_skips_missing(self, _isolate_env):
dg = _load_lib()
assert dg.track(str(_isolate_env / "nope.txt"), "test", silent=True) is False
def test_forget_removes_entry(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "keep.tmp"
p.write_text("x")
dg.track(str(p), "temp", silent=True)
assert dg.forget(str(p)) == 1
assert p.exists() # forget does NOT delete the file
def test_quick_preserves_unexpired_temp(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "fresh.tmp"
p.write_text("x")
dg.track(str(p), "temp", silent=True)
summary = dg.quick()
assert summary["deleted"] == 0
assert p.exists()
def test_quick_preserves_protected_top_level_dirs(self, _isolate_env):
dg = _load_lib()
for d in ("logs", "memories", "sessions", "cron", "cache"):
(_isolate_env / d).mkdir()
dg.quick()
for d in ("logs", "memories", "sessions", "cron", "cache"):
assert (_isolate_env / d).exists(), f"{d}/ should be preserved"
class TestStatus:
def test_empty_status(self, _isolate_env):
dg = _load_lib()
s = dg.status()
assert s["total_tracked"] == 0
assert s["top10"] == []
def test_status_with_entries(self, _isolate_env):
dg = _load_lib()
p = _isolate_env / "big.tmp"
p.write_text("y" * 100)
dg.track(str(p), "temp", silent=True)
s = dg.status()
assert s["total_tracked"] == 1
assert len(s["top10"]) == 1
rendered = dg.format_status(s)
assert "temp" in rendered
assert "big.tmp" in rendered
class TestDryRun:
def test_classifies_by_category(self, _isolate_env):
dg = _load_lib()
test_f = _isolate_env / "test_x.py"
test_f.write_text("x")
big = _isolate_env / "big.bin"
big.write_bytes(b"z" * 10)
dg.track(str(test_f), "test", silent=True)
dg.track(str(big), "other", silent=True)
auto, prompt = dg.dry_run()
# test → auto, other → neither (doesn't hit any rule)
assert any(i["path"] == str(test_f) for i in auto)
# ---------------------------------------------------------------------------
# Plugin hooks tests
# ---------------------------------------------------------------------------
class TestPostToolCallHook:
def test_write_file_test_pattern_tracked(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "test_created.py"
p.write_text("x")
pi._on_post_tool_call(
tool_name="write_file",
args={"path": str(p), "content": "x"},
result="OK",
task_id="t1", session_id="s1",
)
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
data = json.loads(tracked_file.read_text())
assert len(data) == 1
assert data[0]["category"] == "test"
def test_write_file_non_test_not_tracked(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "notes.md"
p.write_text("x")
pi._on_post_tool_call(
tool_name="write_file",
args={"path": str(p), "content": "x"},
result="OK",
task_id="t2", session_id="s2",
)
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
assert not tracked_file.exists() or tracked_file.read_text().strip() == "[]"
def test_terminal_command_picks_up_paths(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "tmp_created.log"
p.write_text("x")
pi._on_post_tool_call(
tool_name="terminal",
args={"command": f"touch {p}"},
result=f"created {p}\n",
task_id="t3", session_id="s3",
)
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
data = json.loads(tracked_file.read_text())
assert any(Path(i["path"]) == p.resolve() for i in data)
def test_ignores_unrelated_tool(self, _isolate_env):
pi = _load_plugin_init()
pi._on_post_tool_call(
tool_name="read_file",
args={"path": str(_isolate_env / "test_x.py")},
result="contents",
task_id="t4", session_id="s4",
)
# read_file should never trigger tracking.
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
assert not tracked_file.exists() or tracked_file.read_text().strip() == "[]"
class TestOnSessionEndHook:
def test_runs_quick_when_test_files_tracked(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "test_cleanup.py"
p.write_text("x")
pi._on_post_tool_call(
tool_name="write_file",
args={"path": str(p), "content": "x"},
result="OK",
task_id="", session_id="s1",
)
assert p.exists()
pi._on_session_end(session_id="s1", completed=True, interrupted=False)
assert not p.exists(), "test file should be auto-deleted"
def test_noop_when_no_test_tracked(self, _isolate_env):
pi = _load_plugin_init()
# Nothing tracked → on_session_end should not raise.
pi._on_session_end(session_id="empty", completed=True, interrupted=False)
# ---------------------------------------------------------------------------
# Slash command
# ---------------------------------------------------------------------------
class TestSlashCommand:
def test_help(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("help")
assert "disk-cleanup" in out
assert "status" in out
def test_status_empty(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("status")
assert "nothing tracked" in out
def test_track_rejects_missing(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash(
f"track {_isolate_env / 'nope.txt'} temp"
)
assert "Not tracked" in out
def test_track_rejects_bad_category(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "a.tmp"
p.write_text("x")
out = pi._handle_slash(f"track {p} banana")
assert "Unknown category" in out
def test_track_and_forget(self, _isolate_env):
pi = _load_plugin_init()
p = _isolate_env / "a.tmp"
p.write_text("x")
out = pi._handle_slash(f"track {p} temp")
assert "Tracked" in out
out = pi._handle_slash(f"forget {p}")
assert "Removed 1" in out
def test_unknown_subcommand(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("foobar")
assert "Unknown subcommand" in out
def test_quick_on_empty(self, _isolate_env):
pi = _load_plugin_init()
out = pi._handle_slash("quick")
assert "Cleaned 0 files" in out
# ---------------------------------------------------------------------------
# Bundled-plugin discovery
# ---------------------------------------------------------------------------
class TestBundledDiscovery:
def _write_enabled_config(self, hermes_home, names):
"""Write plugins.enabled allow-list to config.yaml."""
import yaml
cfg_path = hermes_home / "config.yaml"
cfg_path.write_text(yaml.safe_dump({"plugins": {"enabled": list(names)}}))
def test_disk_cleanup_discovered_but_not_loaded_by_default(self, _isolate_env):
"""Bundled plugins are discovered but NOT loaded without opt-in."""
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
# Discovered — appears in the registry
assert "disk-cleanup" in mgr._plugins
loaded = mgr._plugins["disk-cleanup"]
assert loaded.manifest.source == "bundled"
# But NOT enabled — no hooks or commands registered
assert not loaded.enabled
assert loaded.error and "not enabled" in loaded.error
def test_disk_cleanup_loads_when_enabled(self, _isolate_env):
"""Adding to plugins.enabled activates the bundled plugin."""
self._write_enabled_config(_isolate_env, ["disk-cleanup"])
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
loaded = mgr._plugins["disk-cleanup"]
assert loaded.enabled
assert "post_tool_call" in loaded.hooks_registered
assert "on_session_end" in loaded.hooks_registered
assert "disk-cleanup" in loaded.commands_registered
def test_disabled_beats_enabled(self, _isolate_env):
"""plugins.disabled wins even if the plugin is also in plugins.enabled."""
import yaml
cfg_path = _isolate_env / "config.yaml"
cfg_path.write_text(yaml.safe_dump({
"plugins": {
"enabled": ["disk-cleanup"],
"disabled": ["disk-cleanup"],
}
}))
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
loaded = mgr._plugins["disk-cleanup"]
assert not loaded.enabled
assert loaded.error == "disabled via config"
def test_memory_and_context_engine_subdirs_skipped(self, _isolate_env):
"""Bundled scan must NOT pick up plugins/memory or plugins/context_engine
as top-level plugins — they have their own discovery paths."""
self._write_enabled_config(
_isolate_env, ["memory", "context_engine", "disk-cleanup"]
)
from hermes_cli import plugins as pmod
mgr = pmod.PluginManager()
mgr.discover_and_load()
assert "memory" not in mgr._plugins
assert "context_engine" not in mgr._plugins
+265
View File
@@ -0,0 +1,265 @@
"""Tests for plugins.google_meet.audio_bridge (v2).
Covers the platform gating and pactl / system_profiler plumbing
without actually invoking those tools on the host.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# Linux setup / teardown
# ---------------------------------------------------------------------------
def _linux_pactl_result(stdout: str) -> MagicMock:
"""Build a fake CompletedProcess-ish object for subprocess.run."""
m = MagicMock()
m.stdout = stdout
m.stderr = ""
m.returncode = 0
return m
def test_setup_linux_loads_null_sink_and_virtual_source():
from plugins.google_meet.audio_bridge import AudioBridge
calls: list[list[str]] = []
def _fake_run(argv, **kwargs):
calls.append(list(argv))
# First call = null-sink → module id 42
# Second call = virtual-source → module id 43
if "module-null-sink" in argv:
return _linux_pactl_result("42\n")
if "module-virtual-source" in argv:
return _linux_pactl_result("43\n")
raise AssertionError(f"unexpected pactl invocation: {argv}")
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_fake_run):
br = AudioBridge()
info = br.setup()
# Two pactl load-module calls, in order.
assert len(calls) == 2
assert calls[0][0] == "pactl" and calls[0][1] == "load-module"
assert "module-null-sink" in calls[0]
assert any(a.startswith("sink_name=hermes_meet_sink") for a in calls[0])
assert calls[1][0] == "pactl" and calls[1][1] == "load-module"
assert "module-virtual-source" in calls[1]
assert any(a.startswith("source_name=hermes_meet_src") for a in calls[1])
assert any("master=hermes_meet_sink.monitor" in a for a in calls[1])
# Dict shape.
assert info["platform"] == "linux"
assert info["device_name"] == "hermes_meet_src"
assert info["write_target"] == "hermes_meet_sink"
assert info["sample_rate"] == 48000
assert info["channels"] == 2
assert info["module_ids"] == [42, 43]
# Properties.
assert br.device_name == "hermes_meet_src"
assert br.write_target == "hermes_meet_sink"
def test_teardown_linux_unloads_modules_in_reverse_order():
from plugins.google_meet.audio_bridge import AudioBridge
def _setup_run(argv, **kwargs):
if "module-null-sink" in argv:
return _linux_pactl_result("42\n")
return _linux_pactl_result("43\n")
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_setup_run):
br = AudioBridge()
br.setup()
unload_calls: list[list[str]] = []
def _teardown_run(argv, **kwargs):
unload_calls.append(list(argv))
return _linux_pactl_result("")
with patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_teardown_run):
br.teardown()
# Two unload calls, in reverse order: 43 (virtual-source) then 42 (sink).
assert [c[1] for c in unload_calls] == ["unload-module", "unload-module"]
assert unload_calls[0][2] == "43"
assert unload_calls[1][2] == "42"
# Second teardown is a no-op.
with patch("plugins.google_meet.audio_bridge.subprocess.run") as run_mock:
br.teardown()
run_mock.assert_not_called()
def test_setup_linux_parses_module_id_from_multi_line_output():
"""Some pactl builds include trailing whitespace / notices."""
from plugins.google_meet.audio_bridge import AudioBridge
def _fake_run(argv, **kwargs):
if "module-null-sink" in argv:
return _linux_pactl_result("42 \n")
return _linux_pactl_result("43\n")
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=_fake_run):
br = AudioBridge()
info = br.setup()
assert info["module_ids"] == [42, 43]
def test_setup_linux_pactl_missing_raises_clean_error():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"), \
patch("plugins.google_meet.audio_bridge.subprocess.run",
side_effect=FileNotFoundError("pactl")):
br = AudioBridge()
with pytest.raises(RuntimeError, match="pactl"):
br.setup()
# ---------------------------------------------------------------------------
# macOS setup
# ---------------------------------------------------------------------------
_BH_PRESENT = (
"Audio:\n"
" Devices:\n"
" BlackHole 2ch:\n"
" Manufacturer: Existential Audio\n"
)
_BH_ABSENT = (
"Audio:\n"
" Devices:\n"
" MacBook Pro Microphone:\n"
" Default Input: Yes\n"
)
def test_setup_darwin_returns_blackhole_when_present():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Darwin"), \
patch("plugins.google_meet.audio_bridge.subprocess.check_output",
return_value=_BH_PRESENT) as check:
br = AudioBridge()
info = br.setup()
check.assert_called_once()
argv = check.call_args.args[0]
assert argv[0] == "system_profiler"
assert "SPAudioDataType" in argv
assert info["platform"] == "darwin"
assert info["device_name"] == "BlackHole 2ch"
assert info["write_target"] == "BlackHole 2ch"
assert info["module_ids"] == []
assert info["sample_rate"] == 48000
assert info["channels"] == 2
# teardown is a no-op on darwin (no modules to unload).
with patch("plugins.google_meet.audio_bridge.subprocess.run") as run_mock:
br.teardown()
run_mock.assert_not_called()
def test_setup_darwin_raises_when_blackhole_missing():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Darwin"), \
patch("plugins.google_meet.audio_bridge.subprocess.check_output",
return_value=_BH_ABSENT):
br = AudioBridge()
with pytest.raises(RuntimeError, match="BlackHole"):
br.setup()
# ---------------------------------------------------------------------------
# Windows / unsupported
# ---------------------------------------------------------------------------
def test_setup_windows_raises():
from plugins.google_meet.audio_bridge import AudioBridge
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Windows"):
br = AudioBridge()
with pytest.raises(RuntimeError, match="not supported"):
br.setup()
# ---------------------------------------------------------------------------
# chrome_fake_audio_flags
# ---------------------------------------------------------------------------
def test_chrome_fake_audio_flags_linux():
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Linux"):
flags = chrome_fake_audio_flags(
{"platform": "linux", "device_name": "hermes_meet_src"}
)
assert "--use-fake-ui-for-media-stream" in flags
def test_chrome_fake_audio_flags_darwin():
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Darwin"):
flags = chrome_fake_audio_flags(
{"platform": "darwin", "device_name": "BlackHole 2ch"}
)
assert "--use-fake-ui-for-media-stream" in flags
def test_chrome_fake_audio_flags_windows_raises():
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
with patch("plugins.google_meet.audio_bridge.platform.system",
return_value="Windows"):
with pytest.raises(RuntimeError):
chrome_fake_audio_flags({"platform": "windows"})
def test_property_access_before_setup_raises():
from plugins.google_meet.audio_bridge import AudioBridge
br = AudioBridge()
with pytest.raises(RuntimeError):
_ = br.device_name
with pytest.raises(RuntimeError):
_ = br.write_target
+674
View File
@@ -0,0 +1,674 @@
"""Tests for the google_meet node primitive.
Covers protocol helpers, the file-backed registry, the server's
token-and-dispatch machinery, a mocked client, and the CLI plumbing.
We never open a real socket — websockets.serve / websockets.sync.client
are fully mocked.
"""
from __future__ import annotations
import argparse
import asyncio
import json
from pathlib import Path
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# protocol.py
# ---------------------------------------------------------------------------
def test_protocol_encode_decode_roundtrip():
from plugins.google_meet.node import protocol
msg = protocol.make_request("ping", "tok", {"x": 1}, req_id="abc")
raw = protocol.encode(msg)
out = protocol.decode(raw)
assert out == msg
assert out["type"] == "ping"
assert out["id"] == "abc"
assert out["token"] == "tok"
assert out["payload"] == {"x": 1}
def test_protocol_make_request_autogenerates_id():
from plugins.google_meet.node import protocol
a = protocol.make_request("ping", "tok", {})
b = protocol.make_request("ping", "tok", {})
assert a["id"] != b["id"]
assert len(a["id"]) >= 16 # uuid4 hex
def test_protocol_make_request_rejects_bad_input():
from plugins.google_meet.node import protocol
with pytest.raises(ValueError):
protocol.make_request("", "tok", {})
with pytest.raises(ValueError):
protocol.make_request("unknown_type", "tok", {})
with pytest.raises(ValueError):
protocol.make_request("ping", "tok", "not a dict") # type: ignore[arg-type]
def test_protocol_decode_raises_on_malformed():
from plugins.google_meet.node import protocol
with pytest.raises(ValueError):
protocol.decode("not json at all")
with pytest.raises(ValueError):
protocol.decode("[]") # list, not object
with pytest.raises(ValueError):
protocol.decode(json.dumps({"id": "x"})) # missing type
with pytest.raises(ValueError):
protocol.decode(json.dumps({"type": "ping"})) # missing id
def test_protocol_validate_request_happy_path():
from plugins.google_meet.node import protocol
msg = protocol.make_request("status", "secret", {})
ok, reason = protocol.validate_request(msg, "secret")
assert ok is True
assert reason == ""
def test_protocol_validate_request_rejects_bad_token():
from plugins.google_meet.node import protocol
msg = protocol.make_request("status", "wrong", {})
ok, reason = protocol.validate_request(msg, "right")
assert ok is False
assert "token" in reason.lower()
def test_protocol_validate_request_rejects_unknown_type():
from plugins.google_meet.node import protocol
raw = {"type": "nope", "id": "1", "token": "t", "payload": {}}
ok, reason = protocol.validate_request(raw, "t")
assert ok is False
assert "unknown" in reason.lower()
def test_protocol_validate_request_rejects_missing_id():
from plugins.google_meet.node import protocol
raw = {"type": "ping", "token": "t", "payload": {}}
ok, reason = protocol.validate_request(raw, "t")
assert ok is False
assert "id" in reason.lower()
def test_protocol_validate_request_rejects_non_dict_payload():
from plugins.google_meet.node import protocol
raw = {"type": "ping", "id": "1", "token": "t", "payload": "oops"}
ok, reason = protocol.validate_request(raw, "t")
assert ok is False
def test_protocol_error_envelope_shape():
from plugins.google_meet.node import protocol
err = protocol.make_error("abc", "nope")
assert err == {"type": "error", "id": "abc", "error": "nope"}
# ---------------------------------------------------------------------------
# registry.py
# ---------------------------------------------------------------------------
def test_registry_add_get_roundtrip_persists(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
p = tmp_path / "nodes.json"
r = NodeRegistry(path=p)
r.add("mac", "ws://mac.local:18789", "deadbeef")
# Second instance sees it.
r2 = NodeRegistry(path=p)
entry = r2.get("mac")
assert entry is not None
assert entry["name"] == "mac"
assert entry["url"] == "ws://mac.local:18789"
assert entry["token"] == "deadbeef"
assert "added_at" in entry
def test_registry_get_returns_none_when_missing(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
assert r.get("ghost") is None
def test_registry_remove(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("a", "ws://a", "t")
assert r.remove("a") is True
assert r.get("a") is None
assert r.remove("a") is False # idempotent
def test_registry_list_all_sorted(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("zeta", "ws://z", "t1")
r.add("alpha", "ws://a", "t2")
names = [n["name"] for n in r.list_all()]
assert names == ["alpha", "zeta"]
def test_registry_resolve_auto_picks_single(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("mac", "ws://mac", "t")
picked = r.resolve(None)
assert picked is not None
assert picked["name"] == "mac"
def test_registry_resolve_ambiguous_returns_none(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("a", "ws://a", "t")
r.add("b", "ws://b", "t")
assert r.resolve(None) is None
def test_registry_resolve_empty_returns_none(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
assert r.resolve(None) is None
def test_registry_resolve_by_name(tmp_path):
from plugins.google_meet.node.registry import NodeRegistry
r = NodeRegistry(path=tmp_path / "n.json")
r.add("a", "ws://a", "t")
r.add("b", "ws://b", "t")
picked = r.resolve("b")
assert picked is not None
assert picked["name"] == "b"
assert r.resolve("ghost") is None
def test_registry_defaults_to_hermes_home(tmp_path, monkeypatch):
from plugins.google_meet.node.registry import NodeRegistry
# _isolate_home already set HERMES_HOME to tmp_path/.hermes; the
# registry default path must live inside that tree.
r = NodeRegistry()
r.add("x", "ws://x", "t")
expected = Path(tmp_path) / ".hermes" / "workspace" / "meetings" / "nodes.json"
assert expected.is_file()
# ---------------------------------------------------------------------------
# server.py — token + dispatch
# ---------------------------------------------------------------------------
def test_server_ensure_token_generates_and_persists(tmp_path):
from plugins.google_meet.node.server import NodeServer
p = tmp_path / "tok.json"
s1 = NodeServer(token_path=p)
t1 = s1.ensure_token()
assert isinstance(t1, str) and len(t1) == 32
# Reuse on a fresh instance.
s2 = NodeServer(token_path=p)
t2 = s2.ensure_token()
assert t1 == t2
data = json.loads(p.read_text(encoding="utf-8"))
assert data["token"] == t1
assert "generated_at" in data
def test_server_get_token_is_idempotent(tmp_path):
from plugins.google_meet.node.server import NodeServer
s = NodeServer(token_path=tmp_path / "t.json")
assert s.get_token() == s.get_token()
def _run(coro):
return asyncio.new_event_loop().run_until_complete(coro) if False else asyncio.run(coro)
def test_server_handle_request_rejects_bad_token(tmp_path):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
s = NodeServer(token_path=tmp_path / "t.json")
s.ensure_token()
bad = protocol.make_request("ping", "not-the-token", {})
resp = asyncio.run(s._handle_request(bad))
assert resp["type"] == "error"
assert "token" in resp["error"].lower()
def test_server_handle_request_ping(tmp_path):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
s = NodeServer(token_path=tmp_path / "t.json", display_name="node-x")
tok = s.ensure_token()
req = protocol.make_request("ping", tok, {})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "pong"
assert resp["id"] == req["id"]
assert resp["payload"]["display_name"] == "node-x"
def test_server_handle_request_status_dispatches_to_pm(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
monkeypatch.setattr(pm, "status",
lambda: {"ok": True, "alive": True, "meetingId": "abc"})
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("status", tok, {})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["id"] == req["id"]
assert resp["payload"] == {"ok": True, "alive": True, "meetingId": "abc"}
def test_server_handle_request_start_bot_dispatches(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
captured = {}
def fake_start(**kwargs):
captured.update(kwargs)
return {"ok": True, "pid": 42, "meeting_id": "abc-defg-hij"}
monkeypatch.setattr(pm, "start", fake_start)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("start_bot", tok, {
"url": "https://meet.google.com/abc-defg-hij",
"guest_name": "Bot",
"duration": "30m",
})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["ok"] is True
assert captured["url"] == "https://meet.google.com/abc-defg-hij"
assert captured["guest_name"] == "Bot"
assert captured["duration"] == "30m"
def test_server_handle_request_start_bot_missing_url(tmp_path):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("start_bot", tok, {"guest_name": "x"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "error"
assert "url" in resp["error"]
def test_server_handle_request_stop_dispatches(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
got = {}
def fake_stop(*, reason="requested"):
got["reason"] = reason
return {"ok": True, "reason": reason}
monkeypatch.setattr(pm, "stop", fake_stop)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("stop", tok, {"reason": "user-cancel"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert got["reason"] == "user-cancel"
def test_server_handle_request_transcript(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
got = {}
def fake_transcript(last=None):
got["last"] = last
return {"ok": True, "lines": ["a", "b"], "total": 2}
monkeypatch.setattr(pm, "transcript", fake_transcript)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("transcript", tok, {"last": 5})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["lines"] == ["a", "b"]
assert got["last"] == 5
def test_server_handle_request_say_enqueues_when_active(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
out = tmp_path / "meet-out"
out.mkdir()
monkeypatch.setattr(pm, "_read_active",
lambda: {"pid": 1, "meeting_id": "m", "out_dir": str(out)})
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("say", tok, {"text": "hello"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["ok"] is True
assert resp["payload"]["enqueued"] is True
q = (out / "say_queue.jsonl").read_text(encoding="utf-8").strip().splitlines()
assert len(q) == 1
assert json.loads(q[0])["text"] == "hello"
def test_server_handle_request_say_without_active_still_ok(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
monkeypatch.setattr(pm, "_read_active", lambda: None)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("say", tok, {"text": "hi"})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "response"
assert resp["payload"]["ok"] is True
assert resp["payload"]["enqueued"] is False
def test_server_handle_request_wraps_pm_exceptions(tmp_path, monkeypatch):
from plugins.google_meet.node.server import NodeServer
from plugins.google_meet.node import protocol
from plugins.google_meet import process_manager as pm
def boom():
raise ValueError("kaboom")
monkeypatch.setattr(pm, "status", boom)
s = NodeServer(token_path=tmp_path / "t.json")
tok = s.ensure_token()
req = protocol.make_request("status", tok, {})
resp = asyncio.run(s._handle_request(req))
assert resp["type"] == "error"
assert "kaboom" in resp["error"]
# ---------------------------------------------------------------------------
# client.py
# ---------------------------------------------------------------------------
class _FakeWS:
"""Minimal context-manager stand-in for websockets.sync.client.connect."""
def __init__(self, reply_builder):
self._reply_builder = reply_builder
self.sent = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def send(self, raw):
self.sent.append(raw)
def recv(self, timeout=None):
return self._reply_builder(self.sent[-1])
def _install_fake_ws(monkeypatch, reply_builder):
fake_ws_holder = {}
def _connect(url, **kwargs):
ws = _FakeWS(reply_builder)
fake_ws_holder["ws"] = ws
fake_ws_holder["url"] = url
fake_ws_holder["kwargs"] = kwargs
return ws
# Patch the concrete import site inside client._rpc
import websockets.sync.client as wsc # type: ignore
monkeypatch.setattr(wsc, "connect", _connect)
return fake_ws_holder
def test_client_rpc_sends_correct_envelope_and_parses_response(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
def reply(raw_out):
req = protocol.decode(raw_out)
return protocol.encode(protocol.make_response(req["id"], {"ok": True, "echo": req["type"]}))
holder = _install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://remote:1", "tok123")
out = c._rpc("ping", {"hello": 1})
assert out == {"ok": True, "echo": "ping"}
sent = json.loads(holder["ws"].sent[0])
assert sent["type"] == "ping"
assert sent["token"] == "tok123"
assert sent["payload"] == {"hello": 1}
assert sent["id"] # non-empty
assert holder["url"] == "ws://remote:1"
def test_client_rpc_raises_on_error_envelope(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
def reply(raw_out):
req = protocol.decode(raw_out)
return protocol.encode(protocol.make_error(req["id"], "nope"))
_install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://x", "t")
with pytest.raises(RuntimeError, match="nope"):
c._rpc("ping", {})
def test_client_rpc_raises_on_id_mismatch(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
def reply(raw_out):
return protocol.encode(protocol.make_response("different-id", {"ok": True}))
_install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://x", "t")
with pytest.raises(RuntimeError, match="mismatch"):
c._rpc("ping", {})
def test_client_convenience_methods_hit_correct_types(monkeypatch):
from plugins.google_meet.node.client import NodeClient
from plugins.google_meet.node import protocol
seen = []
def reply(raw_out):
req = protocol.decode(raw_out)
seen.append((req["type"], req["payload"]))
return protocol.encode(protocol.make_response(req["id"], {"ok": True}))
_install_fake_ws(monkeypatch, reply)
c = NodeClient("ws://x", "t")
c.start_bot("https://meet.google.com/a-b-c", guest_name="G", duration="10m")
c.stop()
c.status()
c.transcript(last=3)
c.say("hi")
c.ping()
types = [t for t, _ in seen]
assert types == ["start_bot", "stop", "status", "transcript", "say", "ping"]
# Check specific payload routing
assert seen[0][1]["url"] == "https://meet.google.com/a-b-c"
assert seen[0][1]["guest_name"] == "G"
assert seen[0][1]["duration"] == "10m"
assert seen[3][1]["last"] == 3
assert seen[4][1]["text"] == "hi"
def test_client_init_rejects_bad_args():
from plugins.google_meet.node.client import NodeClient
with pytest.raises(ValueError):
NodeClient("", "t")
with pytest.raises(ValueError):
NodeClient("ws://x", "")
# ---------------------------------------------------------------------------
# cli.py
# ---------------------------------------------------------------------------
def _build_parser():
from plugins.google_meet.node.cli import register_cli
parser = argparse.ArgumentParser(prog="meet-node-test")
register_cli(parser)
return parser
def test_cli_approve_list_remove(capsys):
from plugins.google_meet.node.registry import NodeRegistry
p = _build_parser()
args = p.parse_args(["approve", "mac", "ws://mac:1", "tok"])
rc = args.func(args)
assert rc == 0
assert NodeRegistry().get("mac") is not None
args = p.parse_args(["list"])
rc = args.func(args)
assert rc == 0
out = capsys.readouterr().out
assert "mac" in out
assert "ws://mac:1" in out
args = p.parse_args(["remove", "mac"])
rc = args.func(args)
assert rc == 0
assert NodeRegistry().get("mac") is None
def test_cli_list_empty(capsys):
p = _build_parser()
args = p.parse_args(["list"])
rc = args.func(args)
assert rc == 0
assert "no nodes" in capsys.readouterr().out
def test_cli_remove_missing_returns_nonzero():
p = _build_parser()
args = p.parse_args(["remove", "ghost"])
rc = args.func(args)
assert rc == 1
def test_cli_status_pings_via_node_client(capsys, monkeypatch):
from plugins.google_meet.node.registry import NodeRegistry
from plugins.google_meet.node import cli as node_cli
NodeRegistry().add("mac", "ws://mac:1", "tok")
class _FakeClient:
def __init__(self, url, token):
assert url == "ws://mac:1"
assert token == "tok"
def ping(self):
return {"type": "pong", "display_name": "hermes-meet-node"}
monkeypatch.setattr(node_cli, "NodeClient", _FakeClient)
p = _build_parser()
args = p.parse_args(["status", "mac"])
rc = args.func(args)
assert rc == 0
out = capsys.readouterr().out.strip()
data = json.loads(out)
assert data["ok"] is True
assert data["node"] == "mac"
def test_cli_status_unknown_node_fails(capsys):
p = _build_parser()
args = p.parse_args(["status", "ghost"])
rc = args.func(args)
assert rc == 1
def test_cli_status_reports_client_error(capsys, monkeypatch):
from plugins.google_meet.node.registry import NodeRegistry
from plugins.google_meet.node import cli as node_cli
NodeRegistry().add("mac", "ws://mac:1", "tok")
class _FakeClient:
def __init__(self, url, token):
pass
def ping(self):
raise RuntimeError("connection refused")
monkeypatch.setattr(node_cli, "NodeClient", _FakeClient)
p = _build_parser()
args = p.parse_args(["status", "mac"])
rc = args.func(args)
assert rc == 1
data = json.loads(capsys.readouterr().out.strip())
assert data["ok"] is False
assert "connection refused" in data["error"]
+813
View File
@@ -0,0 +1,813 @@
"""Tests for the google_meet plugin.
Covers the safety-gated pieces that don't require Playwright:
* URL regex — only ``https://meet.google.com/`` URLs pass
* Meeting-id extraction from Meet URLs
* Status / transcript writes round-trip through the file-backed state
* Tool handlers return well-formed JSON under all branches
* Process manager refuses unsafe URLs and clears stale state cleanly
* ``_on_session_end`` hook is defensive (no-ops when no bot active)
Does NOT spawn a real Chromium — we mock ``subprocess.Popen`` where needed.
"""
from __future__ import annotations
import json
import os
import signal
from pathlib import Path
from unittest.mock import patch
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# URL safety gate
# ---------------------------------------------------------------------------
def test_is_safe_meet_url_accepts_standard_meet_codes():
from plugins.google_meet.meet_bot import _is_safe_meet_url
assert _is_safe_meet_url("https://meet.google.com/abc-defg-hij")
assert _is_safe_meet_url("https://meet.google.com/abc-defg-hij?pli=1")
assert _is_safe_meet_url("https://meet.google.com/new")
assert _is_safe_meet_url("https://meet.google.com/lookup/ABC123")
def test_is_safe_meet_url_rejects_non_meet_urls():
from plugins.google_meet.meet_bot import _is_safe_meet_url
# wrong host
assert not _is_safe_meet_url("https://evil.example.com/abc-defg-hij")
# wrong scheme
assert not _is_safe_meet_url("http://meet.google.com/abc-defg-hij")
# malformed code
assert not _is_safe_meet_url("https://meet.google.com/not-a-meet-code")
# subdomain hijack attempts
assert not _is_safe_meet_url("https://meet.google.com.evil.com/abc-defg-hij")
assert not _is_safe_meet_url("https://notmeet.google.com/abc-defg-hij")
# empty / wrong type
assert not _is_safe_meet_url("")
assert not _is_safe_meet_url(None) # type: ignore[arg-type]
assert not _is_safe_meet_url(123) # type: ignore[arg-type]
def test_meeting_id_extraction():
from plugins.google_meet.meet_bot import _meeting_id_from_url
assert _meeting_id_from_url("https://meet.google.com/abc-defg-hij") == "abc-defg-hij"
assert _meeting_id_from_url("https://meet.google.com/abc-defg-hij?pli=1") == "abc-defg-hij"
# fallback for codes we can't parse (e.g. /new before redirect)
fallback = _meeting_id_from_url("https://meet.google.com/new")
assert fallback.startswith("meet-")
# ---------------------------------------------------------------------------
# _BotState — transcript + status file round-trip
# ---------------------------------------------------------------------------
def test_bot_state_dedupes_captions_and_flushes_status(tmp_path):
from plugins.google_meet.meet_bot import _BotState
out = tmp_path / "session"
state = _BotState(out_dir=out, meeting_id="abc-defg-hij",
url="https://meet.google.com/abc-defg-hij")
state.record_caption("Alice", "Hey everyone")
state.record_caption("Alice", "Hey everyone") # dup — ignored
state.record_caption("Bob", "Let's start")
transcript = (out / "transcript.txt").read_text()
assert "Alice: Hey everyone" in transcript
assert "Bob: Let's start" in transcript
# dedup — Alice line appears exactly once
assert transcript.count("Alice: Hey everyone") == 1
status = json.loads((out / "status.json").read_text())
assert status["meetingId"] == "abc-defg-hij"
assert status["transcriptLines"] == 2
assert status["transcriptPath"].endswith("transcript.txt")
def test_bot_state_ignores_blank_text(tmp_path):
from plugins.google_meet.meet_bot import _BotState
state = _BotState(out_dir=tmp_path / "s", meeting_id="x-y-z",
url="https://meet.google.com/x-y-z")
state.record_caption("Alice", "")
state.record_caption("Alice", " ")
state.record_caption("", "text but no speaker")
status = json.loads((tmp_path / "s" / "status.json").read_text())
assert status["transcriptLines"] == 1
# blank-speaker falls back to "Unknown"
assert "Unknown: text but no speaker" in (tmp_path / "s" / "transcript.txt").read_text()
def test_parse_duration():
from plugins.google_meet.meet_bot import _parse_duration
assert _parse_duration("30m") == 30 * 60
assert _parse_duration("2h") == 2 * 3600
assert _parse_duration("90s") == 90
assert _parse_duration("90") == 90
assert _parse_duration("") is None
assert _parse_duration("bogus") is None
# ---------------------------------------------------------------------------
# process_manager — refuses unsafe URLs, manages active pointer
# ---------------------------------------------------------------------------
def test_start_refuses_unsafe_url():
from plugins.google_meet import process_manager as pm
res = pm.start("https://evil.example.com/abc-defg-hij")
assert res["ok"] is False
assert "refusing" in res["error"]
def test_status_reports_no_active_meeting():
from plugins.google_meet import process_manager as pm
assert pm.status() == {"ok": False, "reason": "no active meeting"}
assert pm.transcript() == {"ok": False, "reason": "no active meeting"}
assert pm.stop() == {"ok": False, "reason": "no active meeting"}
def test_start_spawns_subprocess_and_writes_active_pointer(tmp_path):
"""Verify start() wires env vars correctly and records the pid."""
from plugins.google_meet import process_manager as pm
class _FakeProc:
def __init__(self, pid):
self.pid = pid
captured_env = {}
captured_argv = []
def _fake_popen(argv, **kwargs):
captured_argv.extend(argv)
captured_env.update(kwargs.get("env") or {})
return _FakeProc(99999)
with patch.object(pm.subprocess, "Popen", side_effect=_fake_popen):
# Also prevent pid liveness probe from stomping on our real pids
with patch.object(pm, "_pid_alive", return_value=False):
res = pm.start(
"https://meet.google.com/abc-defg-hij",
guest_name="Test Bot",
duration="15m",
)
assert res["ok"] is True
assert res["meeting_id"] == "abc-defg-hij"
assert res["pid"] == 99999
assert captured_env["HERMES_MEET_URL"] == "https://meet.google.com/abc-defg-hij"
assert captured_env["HERMES_MEET_GUEST_NAME"] == "Test Bot"
assert captured_env["HERMES_MEET_DURATION"] == "15m"
# python -m plugins.google_meet.meet_bot
assert any("plugins.google_meet.meet_bot" in a for a in captured_argv)
# .active.json points at the bot
active = pm._read_active()
assert active is not None
assert active["pid"] == 99999
assert active["meeting_id"] == "abc-defg-hij"
def test_transcript_reads_last_n_lines(tmp_path):
from plugins.google_meet import process_manager as pm
meeting_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
meeting_dir.mkdir(parents=True)
(meeting_dir / "transcript.txt").write_text(
"[10:00:00] Alice: one\n"
"[10:00:01] Bob: two\n"
"[10:00:02] Alice: three\n"
)
pm._write_active({
"pid": 0, "meeting_id": "abc-defg-hij",
"out_dir": str(meeting_dir),
"url": "https://meet.google.com/abc-defg-hij",
"started_at": 0,
})
res = pm.transcript(last=2)
assert res["ok"] is True
assert res["total"] == 3
assert len(res["lines"]) == 2
assert res["lines"][-1].endswith("Alice: three")
def test_stop_signals_process_and_clears_pointer(tmp_path):
from plugins.google_meet import process_manager as pm
pm._write_active({
"pid": 11111, "meeting_id": "x-y-z",
"out_dir": str(tmp_path / "x-y-z"),
"url": "https://meet.google.com/x-y-z",
"started_at": 0,
})
alive_seq = iter([True, True, False]) # alive at first, gone after SIGTERM
def _alive(pid):
try:
return next(alive_seq)
except StopIteration:
return False
sent = []
def _kill(pid, sig):
sent.append((pid, sig))
with patch.object(pm, "_pid_alive", side_effect=_alive), \
patch.object(pm.os, "kill", side_effect=_kill), \
patch.object(pm.time, "sleep", lambda _s: None):
res = pm.stop()
assert res["ok"] is True
assert (11111, signal.SIGTERM) in sent
# .active.json cleared
assert pm._read_active() is None
# ---------------------------------------------------------------------------
# Tool handlers — JSON shape + safety gates
# ---------------------------------------------------------------------------
def test_meet_join_handler_missing_url_returns_error():
from plugins.google_meet.tools import handle_meet_join
out = json.loads(handle_meet_join({}))
assert out["success"] is False
assert "url is required" in out["error"]
def test_meet_join_handler_respects_safety_gate():
from plugins.google_meet.tools import handle_meet_join
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=True):
out = json.loads(handle_meet_join({"url": "https://evil.example.com/foo"}))
assert out["success"] is False
assert "refusing" in out["error"]
def test_meet_join_handler_returns_error_when_playwright_missing():
from plugins.google_meet.tools import handle_meet_join
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=False):
out = json.loads(handle_meet_join({"url": "https://meet.google.com/abc-defg-hij"}))
assert out["success"] is False
assert "prerequisites missing" in out["error"]
def test_meet_say_requires_text():
from plugins.google_meet.tools import handle_meet_say
out = json.loads(handle_meet_say({}))
assert out["success"] is False
assert "text is required" in out["error"]
def test_meet_say_no_active_meeting():
from plugins.google_meet.tools import handle_meet_say
out = json.loads(handle_meet_say({"text": "hello everyone"}))
assert out["success"] is False
# Falls through to pm.enqueue_say which reports no active meeting.
assert "no active meeting" in out.get("reason", "")
def test_meet_status_and_transcript_no_active():
from plugins.google_meet.tools import handle_meet_status, handle_meet_transcript
assert json.loads(handle_meet_status({}))["success"] is False
assert json.loads(handle_meet_transcript({}))["success"] is False
def test_meet_leave_no_active():
from plugins.google_meet.tools import handle_meet_leave
out = json.loads(handle_meet_leave({}))
assert out["success"] is False
# ---------------------------------------------------------------------------
# _on_session_end — defensive cleanup
# ---------------------------------------------------------------------------
def test_on_session_end_noop_when_nothing_active():
from plugins.google_meet import _on_session_end
# Should not raise and should not call stop().
with patch("plugins.google_meet.pm.stop") as stop_mock:
_on_session_end()
stop_mock.assert_not_called()
def test_on_session_end_stops_live_bot():
from plugins.google_meet import _on_session_end
from plugins.google_meet import pm
with patch.object(pm, "status", return_value={"ok": True, "alive": True}), \
patch.object(pm, "stop") as stop_mock:
_on_session_end()
stop_mock.assert_called_once()
# ---------------------------------------------------------------------------
# Plugin register() — platform gating + tool registration
# ---------------------------------------------------------------------------
def test_register_refuses_on_windows():
import plugins.google_meet as plugin
calls = {"tools": [], "cli": [], "hooks": []}
class _Ctx:
def register_tool(self, **kw): calls["tools"].append(kw["name"])
def register_cli_command(self, **kw): calls["cli"].append(kw["name"])
def register_hook(self, name, fn): calls["hooks"].append(name)
with patch.object(plugin.platform, "system", return_value="Windows"):
plugin.register(_Ctx())
assert calls == {"tools": [], "cli": [], "hooks": []}
def test_register_wires_tools_cli_and_hook_on_linux():
import plugins.google_meet as plugin
calls = {"tools": [], "cli": [], "hooks": []}
class _Ctx:
def register_tool(self, **kw): calls["tools"].append(kw["name"])
def register_cli_command(self, **kw): calls["cli"].append(kw["name"])
def register_hook(self, name, fn): calls["hooks"].append(name)
with patch.object(plugin.platform, "system", return_value="Linux"):
plugin.register(_Ctx())
assert set(calls["tools"]) == {
"meet_join", "meet_status", "meet_transcript", "meet_leave", "meet_say",
}
assert calls["cli"] == ["meet"]
assert calls["hooks"] == ["on_session_end"]
# ---------------------------------------------------------------------------
# v2: process_manager.enqueue_say + realtime-mode passthrough
# ---------------------------------------------------------------------------
def test_enqueue_say_requires_text():
from plugins.google_meet import process_manager as pm
assert pm.enqueue_say("")["ok"] is False
assert pm.enqueue_say(" ")["ok"] is False
def test_enqueue_say_no_active_meeting():
from plugins.google_meet import process_manager as pm
res = pm.enqueue_say("hi team")
assert res["ok"] is False
assert "no active meeting" in res["reason"]
def test_enqueue_say_rejects_transcribe_mode(tmp_path):
from plugins.google_meet import process_manager as pm
out_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
out_dir.mkdir(parents=True)
pm._write_active({
"pid": 0, "meeting_id": "abc-defg-hij",
"out_dir": str(out_dir), "url": "https://meet.google.com/abc-defg-hij",
"started_at": 0, "mode": "transcribe",
})
res = pm.enqueue_say("hi team")
assert res["ok"] is False
assert "transcribe mode" in res["reason"]
def test_enqueue_say_writes_jsonl_in_realtime_mode():
from plugins.google_meet import process_manager as pm
out_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
out_dir.mkdir(parents=True)
pm._write_active({
"pid": 0, "meeting_id": "abc-defg-hij",
"out_dir": str(out_dir), "url": "https://meet.google.com/abc-defg-hij",
"started_at": 0, "mode": "realtime",
})
res = pm.enqueue_say("hello everyone")
assert res["ok"] is True
assert "enqueued_id" in res
queue = out_dir / "say_queue.jsonl"
assert queue.is_file()
lines = [json.loads(ln) for ln in queue.read_text().splitlines() if ln.strip()]
assert len(lines) == 1
assert lines[0]["text"] == "hello everyone"
def test_start_passes_mode_into_active_record():
from plugins.google_meet import process_manager as pm
class _FakeProc:
def __init__(self, pid): self.pid = pid
with patch.object(pm.subprocess, "Popen", return_value=_FakeProc(12345)), \
patch.object(pm, "_pid_alive", return_value=False):
res = pm.start(
"https://meet.google.com/abc-defg-hij",
mode="realtime",
)
assert res["ok"] is True
assert res["mode"] == "realtime"
assert pm._read_active()["mode"] == "realtime"
def test_start_realtime_env_vars_threaded_through():
from plugins.google_meet import process_manager as pm
class _FakeProc:
def __init__(self, pid): self.pid = pid
captured_env = {}
def _fake_popen(argv, **kwargs):
captured_env.update(kwargs.get("env") or {})
return _FakeProc(11111)
with patch.object(pm.subprocess, "Popen", side_effect=_fake_popen), \
patch.object(pm, "_pid_alive", return_value=False):
pm.start(
"https://meet.google.com/abc-defg-hij",
mode="realtime",
realtime_model="gpt-realtime",
realtime_voice="alloy",
realtime_instructions="Be brief.",
realtime_api_key="sk-test",
)
assert captured_env["HERMES_MEET_MODE"] == "realtime"
assert captured_env["HERMES_MEET_REALTIME_MODEL"] == "gpt-realtime"
assert captured_env["HERMES_MEET_REALTIME_VOICE"] == "alloy"
assert captured_env["HERMES_MEET_REALTIME_INSTRUCTIONS"] == "Be brief."
assert captured_env["HERMES_MEET_REALTIME_KEY"] == "sk-test"
def test_meet_join_accepts_realtime_mode():
from plugins.google_meet.tools import handle_meet_join
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=True), \
patch("plugins.google_meet.tools.pm.start", return_value={"ok": True, "meeting_id": "x-y-z"}) as start_mock:
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"mode": "realtime",
}))
assert out["success"] is True
assert start_mock.call_args.kwargs["mode"] == "realtime"
def test_meet_join_rejects_bad_mode():
from plugins.google_meet.tools import handle_meet_join
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"mode": "bogus",
}))
assert out["success"] is False
assert "mode must be" in out["error"]
# ---------------------------------------------------------------------------
# v3: NodeClient routing from tool handlers
# ---------------------------------------------------------------------------
def test_meet_join_unknown_node_returns_clear_error():
from plugins.google_meet.tools import handle_meet_join
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "my-mac",
}))
assert out["success"] is False
assert "no registered meet node" in out["error"]
def test_meet_join_routes_to_registered_node():
from plugins.google_meet.tools import handle_meet_join
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("my-mac", "ws://1.2.3.4:18789", "tok")
with patch("plugins.google_meet.node.client.NodeClient.start_bot",
return_value={"ok": True, "meeting_id": "a-b-c"}) as call_mock:
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "my-mac",
"mode": "realtime",
}))
assert out["success"] is True
assert out["node"] == "my-mac"
assert call_mock.call_args.kwargs["mode"] == "realtime"
def test_meet_say_routes_to_node():
from plugins.google_meet.tools import handle_meet_say
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("my-mac", "ws://1.2.3.4:18789", "tok")
with patch("plugins.google_meet.node.client.NodeClient.say",
return_value={"ok": True, "enqueued_id": "abc"}) as call_mock:
out = json.loads(handle_meet_say({"text": "hello", "node": "my-mac"}))
assert out["success"] is True
assert out["node"] == "my-mac"
call_mock.assert_called_once_with("hello")
def test_meet_join_auto_node_selects_sole_registered():
from plugins.google_meet.tools import handle_meet_join
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("only-one", "ws://1.2.3.4:18789", "tok")
with patch("plugins.google_meet.node.client.NodeClient.start_bot",
return_value={"ok": True}) as call_mock:
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "auto",
}))
assert out["success"] is True
assert out["node"] == "only-one"
assert call_mock.called
def test_meet_join_auto_node_ambiguous_returns_error():
from plugins.google_meet.tools import handle_meet_join
from plugins.google_meet.node.registry import NodeRegistry
reg = NodeRegistry()
reg.add("a", "ws://1.2.3.4:18789", "tok")
reg.add("b", "ws://5.6.7.8:18789", "tok")
out = json.loads(handle_meet_join({
"url": "https://meet.google.com/abc-defg-hij",
"node": "auto",
}))
assert out["success"] is False
assert "no registered meet node" in out["error"]
def test_cli_register_includes_node_subcommand():
"""`hermes meet` argparse tree includes the node subtree."""
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
# Parse a known-good node invocation to prove the subtree is wired.
ns = parser.parse_args(["node", "list"])
assert ns.meet_command == "node"
assert ns.node_cmd == "list"
def test_cli_join_accepts_mode_and_node_flags():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args([
"join", "https://meet.google.com/abc-defg-hij",
"--mode", "realtime", "--node", "my-mac",
])
assert ns.mode == "realtime"
assert ns.node == "my-mac"
def test_cli_say_subcommand_exists():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args(["say", "hello team", "--node", "my-mac"])
assert ns.text == "hello team"
assert ns.node == "my-mac"
# ---------------------------------------------------------------------------
# v2.1: new _BotState fields + status dict shape
# ---------------------------------------------------------------------------
def test_bot_state_exposes_v2_telemetry_fields(tmp_path):
from plugins.google_meet.meet_bot import _BotState
state = _BotState(out_dir=tmp_path / "s", meeting_id="x-y-z",
url="https://meet.google.com/x-y-z")
# Defaults for the new fields.
status = json.loads((tmp_path / "s" / "status.json").read_text())
for key in (
"realtime", "realtimeReady", "realtimeDevice",
"audioBytesOut", "lastAudioOutAt", "lastBargeInAt",
"joinAttemptedAt", "leaveReason",
):
assert key in status, f"missing v2 telemetry key: {key}"
assert status["realtime"] is False
assert status["realtimeReady"] is False
assert status["audioBytesOut"] == 0
# Setting them flushes them.
state.set(realtime=True, realtime_ready=True, audio_bytes_out=1024,
leave_reason="lobby_timeout")
status = json.loads((tmp_path / "s" / "status.json").read_text())
assert status["realtime"] is True
assert status["realtimeReady"] is True
assert status["audioBytesOut"] == 1024
assert status["leaveReason"] == "lobby_timeout"
# ---------------------------------------------------------------------------
# Admission detection + barge-in helper
# ---------------------------------------------------------------------------
def test_looks_like_human_speaker():
from plugins.google_meet.meet_bot import _looks_like_human_speaker
# Blank, "unknown", "you", and the bot's own name → not human (no barge-in)
for s in ("", " ", "Unknown", "unknown", "You", "you", "Hermes Agent", "hermes agent"):
assert not _looks_like_human_speaker(s, "Hermes Agent"), f"{s!r} should NOT be human"
# Real names → human (barge-in)
for s in ("Alice", "Bob Lee", "@teknium"):
assert _looks_like_human_speaker(s, "Hermes Agent"), f"{s!r} SHOULD be human"
def test_detect_admission_returns_false_on_error():
from plugins.google_meet.meet_bot import _detect_admission
class _FakePage:
def evaluate(self, _js): raise RuntimeError("boom")
assert _detect_admission(_FakePage()) is False
def test_detect_admission_true_when_probe_returns_true():
from plugins.google_meet.meet_bot import _detect_admission
class _FakePage:
def evaluate(self, _js): return True
assert _detect_admission(_FakePage()) is True
def test_detect_denied_returns_false_on_error():
from plugins.google_meet.meet_bot import _detect_denied
class _FakePage:
def evaluate(self, _js): raise RuntimeError("boom")
assert _detect_denied(_FakePage()) is False
# ---------------------------------------------------------------------------
# Realtime session counters + cancel_response (barge-in)
# ---------------------------------------------------------------------------
def test_realtime_session_cancel_response_when_disconnected():
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
# No _ws yet — cancel should no-op and return False.
assert sess.cancel_response() is False
def test_realtime_session_cancel_response_sends_cancel_frame():
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
sent = []
class _FakeWs:
def send(self, msg): sent.append(msg)
sess._ws = _FakeWs()
assert sess.cancel_response() is True
assert len(sent) == 1
import json as _j
envelope = _j.loads(sent[0])
assert envelope == {"type": "response.cancel"}
def test_realtime_session_counters_initialized():
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
assert sess.audio_bytes_out == 0
assert sess.last_audio_out_at is None
# ---------------------------------------------------------------------------
# hermes meet install CLI
# ---------------------------------------------------------------------------
def test_cli_install_subcommand_is_registered():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args(["install"])
assert ns.meet_command == "install"
assert ns.realtime is False
assert ns.yes is False
def test_cli_install_flags_parse():
import argparse
from plugins.google_meet.cli import register_cli
parser = argparse.ArgumentParser(prog="hermes meet")
register_cli(parser)
ns = parser.parse_args(["install", "--realtime", "--yes"])
assert ns.realtime is True
assert ns.yes is True
def test_cmd_install_refuses_windows(capsys):
from plugins.google_meet.cli import _cmd_install
with patch("plugins.google_meet.cli.platform" if False else "platform.system",
return_value="Windows"):
rc = _cmd_install(realtime=False, assume_yes=True)
assert rc == 1
out = capsys.readouterr().out
assert "Windows" in out
def test_cmd_install_runs_pip_and_playwright(capsys):
"""End-to-end wiring: pip + playwright install invoked, returncodes handled."""
from plugins.google_meet.cli import _cmd_install
calls = []
class _FakeRes:
def __init__(self, rc=0): self.returncode = rc
def _fake_run(argv, **kwargs):
calls.append(list(argv))
return _FakeRes(0)
with patch("platform.system", return_value="Linux"), \
patch("subprocess.run", side_effect=_fake_run), \
patch("shutil.which", return_value="/usr/bin/paplay"):
rc = _cmd_install(realtime=False, assume_yes=True)
assert rc == 0
# First invocation: pip install
pip_cmds = [c for c in calls if len(c) > 2 and c[1:4] == ["-m", "pip", "install"]]
assert pip_cmds, f"no pip install run: {calls}"
assert "playwright" in pip_cmds[0]
assert "websockets" in pip_cmds[0]
# Second: playwright install chromium
pw_cmds = [c for c in calls if len(c) > 2 and c[1:4] == ["-m", "playwright", "install"]]
assert pw_cmds, f"no playwright install run: {calls}"
assert "chromium" in pw_cmds[0]
def test_cmd_install_realtime_skips_when_deps_present(capsys):
"""When paplay + pactl are already on PATH, no sudo call happens."""
from plugins.google_meet.cli import _cmd_install
calls = []
class _FakeRes:
def __init__(self, rc=0): self.returncode = rc
def _fake_run(argv, **kwargs):
calls.append(list(argv))
return _FakeRes(0)
with patch("platform.system", return_value="Linux"), \
patch("subprocess.run", side_effect=_fake_run), \
patch("shutil.which", return_value="/usr/bin/paplay"):
rc = _cmd_install(realtime=True, assume_yes=True)
assert rc == 0
# No sudo apt-get call — paplay was already on PATH.
sudo_calls = [c for c in calls if c and c[0] == "sudo"]
assert sudo_calls == [], f"unexpected sudo invocation: {sudo_calls}"
out = capsys.readouterr().out
assert "already installed" in out
+290
View File
@@ -0,0 +1,290 @@
"""Tests for plugins.google_meet.realtime.openai_client (v2).
Uses a scripted fake WebSocket — no network, no API key required.
"""
from __future__ import annotations
import base64
import json
import sys
import types
import pytest
@pytest.fixture(autouse=True)
def _isolate_home(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
yield hermes_home
# ---------------------------------------------------------------------------
# Fake WebSocket
# ---------------------------------------------------------------------------
class _FakeWS:
"""Scripted WS: send() records frames, recv() pops a queue."""
def __init__(self, recv_frames: list):
self.sent: list[dict] = []
self._recv_q: list = list(recv_frames)
self.closed = False
def send(self, payload):
# Always accept str payloads — client encodes JSON with json.dumps.
if isinstance(payload, (bytes, bytearray)):
payload = payload.decode()
self.sent.append(json.loads(payload))
def recv(self, timeout=None): # noqa: ARG002
if not self._recv_q:
raise RuntimeError("fake ws: no more frames")
frame = self._recv_q.pop(0)
if isinstance(frame, dict):
return json.dumps(frame)
return frame
def close(self):
self.closed = True
def _install_fake_websockets(monkeypatch, fake_ws):
"""Install a fake ``websockets.sync.client`` module in sys.modules."""
mod_websockets = types.ModuleType("websockets")
mod_sync = types.ModuleType("websockets.sync")
mod_sync_client = types.ModuleType("websockets.sync.client")
captured = {"url": None, "headers": None, "kwargs": None}
def _connect(url, **kwargs):
captured["url"] = url
captured["kwargs"] = kwargs
captured["headers"] = (
kwargs.get("additional_headers") or kwargs.get("extra_headers")
)
return fake_ws
mod_sync_client.connect = _connect
mod_sync.client = mod_sync_client
mod_websockets.sync = mod_sync
monkeypatch.setitem(sys.modules, "websockets", mod_websockets)
monkeypatch.setitem(sys.modules, "websockets.sync", mod_sync)
monkeypatch.setitem(sys.modules, "websockets.sync.client", mod_sync_client)
return captured
# ---------------------------------------------------------------------------
# connect()
# ---------------------------------------------------------------------------
def test_connect_sends_session_update_with_voice_and_instructions(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
ws = _FakeWS(recv_frames=[])
captured = _install_fake_websockets(monkeypatch, ws)
sess = RealtimeSession(
api_key="sk-test",
model="gpt-realtime",
voice="verse",
instructions="Be brief.",
)
sess.connect()
# Auth + beta headers set.
assert captured["url"].startswith("wss://api.openai.com/v1/realtime")
assert "model=gpt-realtime" in captured["url"]
headers = captured["headers"] or []
hdict = dict(headers)
assert hdict.get("Authorization") == "Bearer sk-test"
assert hdict.get("OpenAI-Beta") == "realtime=v1"
# First frame sent must be session.update with the right shape.
assert len(ws.sent) == 1
update = ws.sent[0]
assert update["type"] == "session.update"
s = update["session"]
assert s["voice"] == "verse"
assert s["instructions"] == "Be brief."
assert set(s["modalities"]) == {"audio", "text"}
assert s["output_audio_format"] == "pcm16"
assert s["input_audio_format"] == "pcm16"
# ---------------------------------------------------------------------------
# speak()
# ---------------------------------------------------------------------------
def test_speak_sends_create_and_response_and_writes_audio(monkeypatch, tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSession
audio_bytes = b"\x01\x02\x03\x04PCM!"
b64 = base64.b64encode(audio_bytes).decode()
recv_frames = [
{"type": "response.created"},
{"type": "response.audio.delta", "delta": b64},
{"type": "response.audio.delta", "delta": base64.b64encode(b"more").decode()},
{"type": "response.done"},
]
ws = _FakeWS(recv_frames=recv_frames)
_install_fake_websockets(monkeypatch, ws)
sink = tmp_path / "out.pcm"
sess = RealtimeSession(api_key="sk-test", audio_sink_path=sink)
sess.connect()
result = sess.speak("Hello everyone.")
# Frames sent after session.update: conversation.item.create then response.create.
types_sent = [f["type"] for f in ws.sent]
assert types_sent == ["session.update", "conversation.item.create", "response.create"]
item = ws.sent[1]["item"]
assert item["role"] == "user"
assert item["content"][0]["type"] == "input_text"
assert item["content"][0]["text"] == "Hello everyone."
resp = ws.sent[2]["response"]
assert resp["modalities"] == ["audio"]
# Audio file got decoded + appended bytes.
data = sink.read_bytes()
assert data == audio_bytes + b"more"
assert result["ok"] is True
assert result["bytes_written"] == len(audio_bytes) + len(b"more")
assert result["duration_ms"] >= 0.0
def test_speak_raises_on_error_frame(monkeypatch, tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSession
ws = _FakeWS(recv_frames=[
{"type": "response.created"},
{"type": "error", "error": {"message": "bad juju"}},
])
_install_fake_websockets(monkeypatch, ws)
sess = RealtimeSession(api_key="sk-test", audio_sink_path=tmp_path / "o.pcm")
sess.connect()
with pytest.raises(RuntimeError, match="bad juju"):
sess.speak("hi")
def test_speak_without_connect_raises(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
sess = RealtimeSession(api_key="sk-test")
with pytest.raises(RuntimeError, match="connect"):
sess.speak("hi")
def test_close_is_idempotent_and_closes_ws(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
ws = _FakeWS(recv_frames=[])
_install_fake_websockets(monkeypatch, ws)
sess = RealtimeSession(api_key="sk-test")
sess.connect()
sess.close()
assert ws.closed is True
# Second close is a no-op.
sess.close()
# ---------------------------------------------------------------------------
# websockets dependency missing
# ---------------------------------------------------------------------------
def test_connect_raises_clean_error_when_websockets_missing(monkeypatch):
from plugins.google_meet.realtime.openai_client import RealtimeSession
# Make `import websockets.sync.client` fail.
monkeypatch.setitem(sys.modules, "websockets", None)
monkeypatch.setitem(sys.modules, "websockets.sync", None)
monkeypatch.setitem(sys.modules, "websockets.sync.client", None)
sess = RealtimeSession(api_key="sk-test")
with pytest.raises(RuntimeError, match="pip install websockets"):
sess.connect()
# ---------------------------------------------------------------------------
# RealtimeSpeaker
# ---------------------------------------------------------------------------
class _StubSession:
def __init__(self):
self.spoken: list[str] = []
def speak(self, text, timeout=30.0): # noqa: ARG002
self.spoken.append(text)
return {"ok": True, "bytes_written": len(text), "duration_ms": 1.0}
def test_speaker_run_until_stopped_processes_queue(tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
queue = tmp_path / "queue.jsonl"
processed = tmp_path / "processed.jsonl"
queue.write_text(
json.dumps({"id": "a", "text": "hello one"}) + "\n"
+ json.dumps({"id": "b", "text": "hello two"}) + "\n"
)
stub = _StubSession()
speaker = RealtimeSpeaker(stub, queue_path=queue, processed_path=processed)
# Stop once the queue is empty.
def _stop():
return queue.exists() and queue.read_text().strip() == ""
speaker.run_until_stopped(_stop, poll_interval=0.01)
assert stub.spoken == ["hello one", "hello two"]
# Processed file has both entries, in order.
lines = [json.loads(l) for l in processed.read_text().splitlines() if l.strip()]
assert [l["id"] for l in lines] == ["a", "b"]
assert all(l["result"]["ok"] for l in lines)
# Queue is empty (possibly empty string) after processing.
assert queue.read_text().strip() == ""
def test_speaker_exits_immediately_when_stop_fn_true(tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
queue = tmp_path / "q.jsonl"
queue.write_text(json.dumps({"id": "x", "text": "never spoken"}) + "\n")
stub = _StubSession()
speaker = RealtimeSpeaker(stub, queue_path=queue)
speaker.run_until_stopped(lambda: True, poll_interval=0.01)
assert stub.spoken == []
def test_speaker_drops_line_without_processed_path_when_none(tmp_path):
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
queue = tmp_path / "q.jsonl"
queue.write_text(json.dumps({"id": "only", "text": "once"}) + "\n")
stub = _StubSession()
speaker = RealtimeSpeaker(stub, queue_path=queue, processed_path=None)
def _stop():
return queue.read_text().strip() == ""
speaker.run_until_stopped(_stop, poll_interval=0.01)
assert stub.spoken == ["once"]
assert queue.read_text().strip() == ""
+291
View File
@@ -0,0 +1,291 @@
"""Tests for Kanban task file attachments (#35338).
Covers three layers:
* ``hermes_cli.kanban_db`` accessors (add/list/get/delete + path helpers)
* the dashboard REST surface (upload / list / download / delete)
* worker-context surfacing so a kanban worker sees the absolute paths
The plugin router is attached to a bare FastAPI app — same approach as
``test_kanban_dashboard_plugin.py`` — so we exercise the real HTTP path
(multipart upload, streaming download) without the whole dashboard.
"""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from hermes_cli import kanban_db as kb
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _load_plugin_router():
repo_root = Path(__file__).resolve().parents[2]
plugin_file = repo_root / "plugins" / "kanban" / "dashboard" / "plugin_api.py"
assert plugin_file.exists(), f"plugin file missing: {plugin_file}"
spec = importlib.util.spec_from_file_location(
"hermes_dashboard_plugin_kanban_attach_test", plugin_file,
)
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = mod
spec.loader.exec_module(mod)
return mod.router
@pytest.fixture
def kanban_home(tmp_path, monkeypatch):
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
kb.init_db()
return home
@pytest.fixture
def client(kanban_home):
app = FastAPI()
app.include_router(_load_plugin_router(), prefix="/api/plugins/kanban")
return TestClient(app)
def _make_task(conn, title="t") -> str:
return kb.create_task(conn, title=title)
# ---------------------------------------------------------------------------
# DB-layer accessors
# ---------------------------------------------------------------------------
def test_add_list_get_delete_attachment(kanban_home, tmp_path):
conn = kb.connect()
try:
task_id = _make_task(conn)
# Write a real blob under the per-task dir so delete can unlink it.
dest_dir = kb.task_attachments_dir(task_id)
dest_dir.mkdir(parents=True, exist_ok=True)
blob = dest_dir / "source.pdf"
blob.write_bytes(b"%PDF-1.4 fake")
att_id = kb.add_attachment(
conn,
task_id,
filename="source.pdf",
stored_path=str(blob),
content_type="application/pdf",
size=blob.stat().st_size,
uploaded_by="tester",
)
assert att_id > 0
atts = kb.list_attachments(conn, task_id)
assert len(atts) == 1
a = atts[0]
assert a.filename == "source.pdf"
assert a.content_type == "application/pdf"
assert a.size == len(b"%PDF-1.4 fake")
assert a.uploaded_by == "tester"
assert a.stored_path == str(blob)
got = kb.get_attachment(conn, att_id)
assert got is not None and got.id == att_id
removed = kb.delete_attachment(conn, att_id)
assert removed is not None and removed.id == att_id
assert kb.list_attachments(conn, task_id) == []
assert not blob.exists(), "delete should unlink the on-disk blob"
assert kb.get_attachment(conn, att_id) is None
finally:
conn.close()
def test_add_attachment_rejects_unknown_task(kanban_home):
conn = kb.connect()
try:
with pytest.raises(ValueError):
kb.add_attachment(
conn, "t_doesnotexist", filename="x.txt", stored_path="/tmp/x.txt"
)
finally:
conn.close()
def test_add_attachment_appends_event(kanban_home):
conn = kb.connect()
try:
task_id = _make_task(conn)
kb.add_attachment(
conn, task_id, filename="a.txt", stored_path="/tmp/a.txt", size=3
)
kinds = [e.kind for e in kb.list_events(conn, task_id)]
assert "attached" in kinds
finally:
conn.close()
def test_delete_attachment_missing_returns_none(kanban_home):
conn = kb.connect()
try:
assert kb.delete_attachment(conn, 999999) is None
finally:
conn.close()
def test_attachments_root_is_per_board(kanban_home, monkeypatch):
# default board uses <root>/kanban/attachments
default_root = kb.attachments_root(board="default")
assert default_root.name == "attachments"
# a named board nests under its board dir
monkeypatch.delenv("HERMES_KANBAN_ATTACHMENTS_ROOT", raising=False)
named = kb.attachments_root(board="default")
assert named == default_root
def test_attachments_root_env_override(kanban_home, monkeypatch, tmp_path):
override = tmp_path / "custom-attach"
monkeypatch.setenv("HERMES_KANBAN_ATTACHMENTS_ROOT", str(override))
assert kb.attachments_root() == override
assert kb.task_attachments_dir("t_abc") == override / "t_abc"
# ---------------------------------------------------------------------------
# Worker context surfacing
# ---------------------------------------------------------------------------
def test_worker_context_lists_attachments_with_absolute_path(kanban_home):
conn = kb.connect()
try:
task_id = _make_task(conn, title="translate PDF")
dest_dir = kb.task_attachments_dir(task_id)
dest_dir.mkdir(parents=True, exist_ok=True)
blob = dest_dir / "manual.pdf"
blob.write_bytes(b"data")
kb.add_attachment(
conn,
task_id,
filename="manual.pdf",
stored_path=str(blob.resolve()),
content_type="application/pdf",
size=4,
)
ctx = kb.build_worker_context(conn, task_id)
assert "## Attachments" in ctx
assert "manual.pdf" in ctx
# The absolute path must appear so the worker can read_file it.
assert str(blob.resolve()) in ctx
finally:
conn.close()
def test_worker_context_no_attachments_section_when_empty(kanban_home):
conn = kb.connect()
try:
task_id = _make_task(conn)
ctx = kb.build_worker_context(conn, task_id)
assert "## Attachments" not in ctx
finally:
conn.close()
# ---------------------------------------------------------------------------
# REST surface — upload / list / download / delete round-trip
# ---------------------------------------------------------------------------
def _create_task_via_api(client) -> str:
r = client.post("/api/plugins/kanban/tasks", json={"title": "x"})
assert r.status_code == 200, r.text
return r.json()["task"]["id"]
def test_upload_list_download_delete_roundtrip(client):
task_id = _create_task_via_api(client)
content = b"hello attachment world"
# Upload
r = client.post(
f"/api/plugins/kanban/tasks/{task_id}/attachments",
files={"file": ("notes.txt", content, "text/plain")},
)
assert r.status_code == 200, r.text
att = r.json()["attachment"]
assert att["filename"] == "notes.txt"
assert att["size"] == len(content)
att_id = att["id"]
# List (drawer also embeds it in GET /tasks/:id)
r = client.get(f"/api/plugins/kanban/tasks/{task_id}/attachments")
assert r.status_code == 200
assert [a["filename"] for a in r.json()["attachments"]] == ["notes.txt"]
detail = client.get(f"/api/plugins/kanban/tasks/{task_id}").json()
assert "attachments" in detail
assert len(detail["attachments"]) == 1
# Download streams the exact bytes back
r = client.get(f"/api/plugins/kanban/attachments/{att_id}")
assert r.status_code == 200
assert r.content == content
# Delete removes the row and the file
r = client.delete(f"/api/plugins/kanban/attachments/{att_id}")
assert r.status_code == 200
assert client.get(f"/api/plugins/kanban/attachments/{att_id}").status_code == 404
assert client.get(
f"/api/plugins/kanban/tasks/{task_id}/attachments"
).json()["attachments"] == []
def test_upload_sanitizes_traversal_filename(client):
task_id = _create_task_via_api(client)
r = client.post(
f"/api/plugins/kanban/tasks/{task_id}/attachments",
files={"file": ("../../../../etc/passwd", b"x", "text/plain")},
)
assert r.status_code == 200, r.text
stored_path = r.json()["attachment"]["stored_path"]
# The leaf name only; never escapes the per-task attachments dir.
assert Path(stored_path).name == "passwd"
task_dir = kb.task_attachments_dir(task_id).resolve()
assert Path(stored_path).resolve().is_relative_to(task_dir)
def test_upload_name_collision_gets_suffixed(client):
task_id = _create_task_via_api(client)
for _ in range(2):
r = client.post(
f"/api/plugins/kanban/tasks/{task_id}/attachments",
files={"file": ("dup.txt", b"a", "text/plain")},
)
assert r.status_code == 200, r.text
names = sorted(
a["filename"]
for a in client.get(
f"/api/plugins/kanban/tasks/{task_id}/attachments"
).json()["attachments"]
)
assert names == ["dup (1).txt", "dup.txt"]
def test_upload_unknown_task_404(client):
r = client.post(
"/api/plugins/kanban/tasks/t_nope/attachments",
files={"file": ("x.txt", b"x", "text/plain")},
)
assert r.status_code == 404
def test_download_unknown_attachment_404(client):
assert client.get("/api/plugins/kanban/attachments/424242").status_code == 404
File diff suppressed because it is too large Load Diff
+440
View File
@@ -0,0 +1,440 @@
"""Tests for kanban worker/runs read endpoints.
Covers:
GET /workers/active
GET /runs/{run_id}
GET /runs/{run_id}/inspect
POST /runs/{run_id}/terminate
"""
from __future__ import annotations
import importlib.util
import secrets
import sys
import time
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from hermes_cli import kanban_db as kb
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _load_plugin_router():
"""Dynamically load plugins/kanban/dashboard/plugin_api.py and return its router."""
repo_root = Path(__file__).resolve().parents[2]
plugin_file = repo_root / "plugins" / "kanban" / "dashboard" / "plugin_api.py"
assert plugin_file.exists(), f"plugin file missing: {plugin_file}"
mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
# Re-use a cached module if already loaded to avoid duplicate-router issues.
if mod_name in sys.modules:
return sys.modules[mod_name].router
spec = importlib.util.spec_from_file_location(mod_name, plugin_file)
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
sys.modules[mod_name] = mod
spec.loader.exec_module(mod)
return mod.router
@pytest.fixture
def kanban_home(tmp_path, monkeypatch):
"""Isolated HERMES_HOME with an empty kanban DB."""
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
kb.init_db()
return home
@pytest.fixture
def client(kanban_home):
app = FastAPI()
app.include_router(_load_plugin_router(), prefix="/api/plugins/kanban")
return TestClient(app)
def _insert_run(conn, task_id, *, worker_pid=None, ended_at=None):
"""Insert a task_runs row directly (bypassing claim machinery) and return run_id."""
lock = secrets.token_hex(8)
future = int(time.time()) + 3600
cur = conn.execute(
"INSERT INTO task_runs "
"(task_id, status, claim_lock, claim_expires, worker_pid, started_at, ended_at) "
"VALUES (?, 'running', ?, ?, ?, ?, ?)",
(task_id, lock, future, worker_pid, int(time.time()), ended_at),
)
conn.commit()
return cur.lastrowid
# ---------------------------------------------------------------------------
# GET /workers/active
# ---------------------------------------------------------------------------
def test_workers_active_empty_board(client):
"""Board with no running tasks returns an empty workers list."""
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
body = r.json()
assert body["workers"] == []
assert body["count"] == 0
assert "checked_at" in body
def test_workers_active_with_running_task(client):
"""A running task with an open run row and worker_pid appears in the list."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="active-worker", assignee="alice")
conn.execute(
"UPDATE tasks SET status='running' WHERE id=?", (task_id,),
)
_insert_run(conn, task_id, worker_pid=12345)
finally:
conn.close()
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
body = r.json()
assert body["count"] == 1
w = body["workers"][0]
assert w["task_id"] == task_id
assert w["worker_pid"] == 12345
assert w["task_status"] == "running"
assert w["task_title"] == "active-worker"
assert w["task_assignee"] == "alice"
def test_workers_active_excludes_ended_runs(client):
"""Runs with ended_at set are excluded even if task is running."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ended-run", assignee="bob")
conn.execute("UPDATE tasks SET status='running' WHERE id=?", (task_id,))
_insert_run(conn, task_id, worker_pid=99999, ended_at=int(time.time()) - 60)
finally:
conn.close()
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
assert r.json()["count"] == 0
def test_workers_active_excludes_runs_without_pid(client):
"""Runs with no worker_pid are not considered active workers."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="no-pid", assignee="carol")
conn.execute("UPDATE tasks SET status='running' WHERE id=?", (task_id,))
_insert_run(conn, task_id, worker_pid=None)
finally:
conn.close()
r = client.get("/api/plugins/kanban/workers/active")
assert r.status_code == 200
assert r.json()["count"] == 0
# ---------------------------------------------------------------------------
# GET /runs/{run_id}
# ---------------------------------------------------------------------------
def test_get_run_404_unknown_id(client):
"""Non-existent run_id returns 404."""
r = client.get("/api/plugins/kanban/runs/999999")
assert r.status_code == 404
assert "999999" in r.json()["detail"]
def test_get_run_ok(client):
"""Existing run row returns 200 with expected shape."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="run-lookup", assignee="dave")
run_id = _insert_run(conn, task_id, worker_pid=55555)
finally:
conn.close()
r = client.get(f"/api/plugins/kanban/runs/{run_id}")
assert r.status_code == 200
body = r.json()
assert "run" in body
run = body["run"]
assert run["id"] == run_id
assert run["task_id"] == task_id
assert run["worker_pid"] == 55555
assert run["ended_at"] is None
# ---------------------------------------------------------------------------
# GET /runs/{run_id}/inspect
# ---------------------------------------------------------------------------
def test_inspect_run_404(client):
"""Non-existent run_id returns 404."""
r = client.get("/api/plugins/kanban/runs/888888/inspect")
assert r.status_code == 404
def test_inspect_run_already_ended(client):
"""Run with ended_at set returns alive=false with reason."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ended", assignee="eve")
run_id = _insert_run(conn, task_id, worker_pid=11111, ended_at=int(time.time()) - 10)
finally:
conn.close()
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is False
assert "ended" in body["reason"]
def test_inspect_run_no_pid(client):
"""Run with no worker_pid returns alive=false with reason."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="no-pid-inspect", assignee="frank")
run_id = _insert_run(conn, task_id, worker_pid=None)
finally:
conn.close()
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is False
assert "worker_pid" in body["reason"]
def test_inspect_run_dead_pid(client, monkeypatch):
"""Run with a non-existent PID returns alive=false via psutil.NoSuchProcess."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="dead-pid", assignee="grace")
run_id = _insert_run(conn, task_id, worker_pid=999999)
finally:
conn.close()
# Mock psutil to raise NoSuchProcess for any PID.
mock_psutil = MagicMock()
mock_psutil.NoSuchProcess = Exception
mock_psutil.AccessDenied = PermissionError
def _raise_no_such(*args, **kwargs):
raise mock_psutil.NoSuchProcess("no such process")
mock_psutil.Process = _raise_no_such
# Patch the module-level _psutil in the loaded plugin module.
plugin_mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
plugin_mod = sys.modules.get(plugin_mod_name)
if plugin_mod is not None:
monkeypatch.setattr(plugin_mod, "_psutil", mock_psutil)
else:
pytest.skip("plugin module not yet loaded")
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is False
assert body["pid"] == 999999
assert "not found" in body["reason"]
def test_inspect_run_live_pid(client, monkeypatch):
"""Run with a live PID returns alive=true with psutil fields."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="live-pid", assignee="heidi")
run_id = _insert_run(conn, task_id, worker_pid=12345)
finally:
conn.close()
# Build a realistic mock psutil.
mock_psutil = MagicMock()
mock_psutil.NoSuchProcess = type("NoSuchProcess", (Exception,), {})
mock_psutil.AccessDenied = type("AccessDenied", (Exception,), {})
fake_mem = MagicMock()
fake_mem.rss = 1024 * 1024 * 50 # 50 MB
fake_mem.vms = 1024 * 1024 * 200
fake_proc = MagicMock()
fake_proc.as_dict.return_value = {
"cpu_percent": 3.5,
"memory_info": fake_mem,
"num_threads": 4,
"status": "sleeping",
"create_time": time.time() - 300,
"cmdline": ["python", "-m", "hermes"],
}
fake_proc.num_fds.return_value = 12
mock_psutil.Process.return_value = fake_proc
plugin_mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
plugin_mod = sys.modules.get(plugin_mod_name)
if plugin_mod is not None:
monkeypatch.setattr(plugin_mod, "_psutil", mock_psutil)
else:
pytest.skip("plugin module not yet loaded")
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
assert r.status_code == 200
body = r.json()
assert body["alive"] is True
assert body["pid"] == 12345
assert body["cpu_percent"] == 3.5
assert body["memory_rss_bytes"] == fake_mem.rss
assert body["num_threads"] == 4
assert body["status"] == "sleeping"
# ---------------------------------------------------------------------------
# POST /runs/{run_id}/terminate
# ---------------------------------------------------------------------------
def _setup_running_task_with_run(conn, *, title, assignee, worker_pid):
"""Create a task in 'running' state with a matching open task_runs row.
Mirrors what dispatcher_claim does: stamps tasks.status='running',
tasks.claim_lock, tasks.worker_pid; inserts task_runs row with the
same claim_lock so reclaim_task's preconditions are satisfied.
"""
task_id = kb.create_task(conn, title=title, assignee=assignee)
lock = secrets.token_hex(8)
future = int(time.time()) + 3600
conn.execute(
"UPDATE tasks SET status='running', claim_lock=?, "
"claim_expires=?, worker_pid=? WHERE id=?",
(lock, future, worker_pid, task_id),
)
cur = conn.execute(
"INSERT INTO task_runs "
"(task_id, status, claim_lock, claim_expires, worker_pid, started_at) "
"VALUES (?, 'running', ?, ?, ?, ?)",
(task_id, lock, future, worker_pid, int(time.time())),
)
conn.commit()
return task_id, cur.lastrowid
def test_terminate_run_404_unknown_id(client):
"""POST to unknown run_id returns 404."""
r = client.post(
"/api/plugins/kanban/runs/777777/terminate",
json={"reason": "test"},
)
assert r.status_code == 404
assert "777777" in r.json()["detail"]
def test_terminate_run_409_already_ended(client):
"""POST against a run with ended_at set returns 409."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ended-terminate", assignee="ivy")
run_id = _insert_run(
conn, task_id, worker_pid=22222, ended_at=int(time.time()) - 30,
)
finally:
conn.close()
r = client.post(
f"/api/plugins/kanban/runs/{run_id}/terminate",
json={"reason": "too late"},
)
assert r.status_code == 409
assert "already ended" in r.json()["detail"]
def test_terminate_run_ok(client, monkeypatch):
"""Happy path: live run is terminated, signal fn invoked, reason recorded."""
conn = kb.connect()
try:
task_id, run_id = _setup_running_task_with_run(
conn, title="kill-me", assignee="jane", worker_pid=33333,
)
finally:
conn.close()
# Capture signal calls so we don't actually SIGTERM a random PID.
sent = []
def _fake_terminate(pid, prev_lock, *, signal_fn=None):
sent.append((pid, prev_lock))
return {"signal": "SIGTERM", "delivered": True}
monkeypatch.setattr(kb, "_terminate_reclaimed_worker", _fake_terminate)
r = client.post(
f"/api/plugins/kanban/runs/{run_id}/terminate",
json={"reason": "operator abort"},
)
assert r.status_code == 200, r.text
body = r.json()
assert body == {"ok": True, "run_id": run_id, "task_id": task_id}
assert sent == [(33333, sent[0][1])]
assert sent[0][1] is not None # claim_lock was non-null
# Task is back to ready, claim cleared.
conn = kb.connect()
try:
row = conn.execute(
"SELECT status, claim_lock, worker_pid FROM tasks WHERE id=?",
(task_id,),
).fetchone()
finally:
conn.close()
assert row["status"] == "ready"
assert row["claim_lock"] is None
assert row["worker_pid"] is None
def test_terminate_run_409_task_not_reclaimable(client, monkeypatch):
"""Open run row whose task is no longer claimable returns 409."""
conn = kb.connect()
try:
task_id = kb.create_task(conn, title="ghost-run", assignee="ken")
# Task left in default 'ready' state with no claim_lock — task_run
# exists but reclaim_task will refuse because status != 'running'
# and claim_lock is NULL.
run_id = _insert_run(conn, task_id, worker_pid=44444)
finally:
conn.close()
# Make sure no signal is ever sent on this code path.
def _boom(*a, **k):
raise AssertionError("_terminate_reclaimed_worker should not be called")
monkeypatch.setattr(kb, "_terminate_reclaimed_worker", _boom)
r = client.post(
f"/api/plugins/kanban/runs/{run_id}/terminate",
json={"reason": "stale"},
)
assert r.status_code == 409
assert "reclaimable" in r.json()["detail"]
def test_terminate_run_accepts_empty_body(client):
"""Empty JSON body (no reason) is still accepted; falls through to 404."""
r = client.post(
"/api/plugins/kanban/runs/666666/terminate",
json={},
)
# 404 because run doesn't exist — what we're asserting here is that
# the endpoint doesn't 422 on a missing 'reason' field.
assert r.status_code == 404
+706
View File
@@ -0,0 +1,706 @@
"""Tests for the bundled observability/langfuse plugin."""
from __future__ import annotations
import importlib
import logging
import sys
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
PLUGIN_DIR = REPO_ROOT / "plugins" / "observability" / "langfuse"
# ---------------------------------------------------------------------------
# Manifest + layout
# ---------------------------------------------------------------------------
class TestManifest:
def test_plugin_directory_exists(self):
assert PLUGIN_DIR.is_dir()
assert (PLUGIN_DIR / "plugin.yaml").exists()
assert (PLUGIN_DIR / "__init__.py").exists()
def test_manifest_fields(self):
data = yaml.safe_load((PLUGIN_DIR / "plugin.yaml").read_text())
assert data["name"] == "langfuse"
assert data["version"]
# All six hooks the plugin implements.
assert set(data["hooks"]) == {
"pre_api_request", "post_api_request",
"pre_llm_call", "post_llm_call",
"pre_tool_call", "post_tool_call",
}
# Required env vars are the user-facing HERMES_ prefixed keys.
assert "HERMES_LANGFUSE_PUBLIC_KEY" in data["requires_env"]
assert "HERMES_LANGFUSE_SECRET_KEY" in data["requires_env"]
# ---------------------------------------------------------------------------
# Plugin discovery: langfuse is opt-in (not loaded unless explicitly enabled).
# This guards against someone accidentally re-introducing a per-hook
# load_config() gate or making the plugin auto-load.
# ---------------------------------------------------------------------------
class TestDiscovery:
def test_plugin_is_discovered_as_standalone_opt_in(self, tmp_path, monkeypatch):
"""Scanner should find the plugin but NOT load it by default."""
from hermes_cli import plugins as plugins_mod
# Isolated HERMES_HOME so we don't read the developer's config.yaml.
home = tmp_path / ".hermes"
home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
manager = plugins_mod.PluginManager()
manager.discover_and_load()
# observability/langfuse appears in the plugin registry …
loaded = manager._plugins.get("observability/langfuse")
assert loaded is not None, "plugin not discovered"
# … but is not loaded (opt-in default → no config.yaml means nothing enabled)
assert loaded.enabled is False
assert "not enabled" in (loaded.error or "").lower()
# ---------------------------------------------------------------------------
# Runtime gate: _get_langfuse() returns None and caches _INIT_FAILED when
# credentials are missing. Guards against regressing toward the rejected
# per-hook load_config() design.
# ---------------------------------------------------------------------------
class TestRuntimeGate:
def _fresh_plugin(self):
"""Import the plugin module fresh (clears any cached client)."""
mod_name = "plugins.observability.langfuse"
sys.modules.pop(mod_name, None)
return importlib.import_module(mod_name)
def test_get_langfuse_returns_none_without_credentials(self, monkeypatch):
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
langfuse_plugin = self._fresh_plugin()
assert langfuse_plugin._get_langfuse() is None
def test_get_langfuse_caches_failure_no_config_load(self, monkeypatch):
"""A miss must be cached — no per-hook config.yaml reads, no env re-reads."""
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
langfuse_plugin = self._fresh_plugin()
# Prime the cache with one call.
assert langfuse_plugin._get_langfuse() is None
# Now block os.environ.get — a correctly-cached plugin must not
# touch env again.
import os
called = {"n": 0}
real_get = os.environ.get
def tracking_get(key, default=None):
if key.startswith(("HERMES_LANGFUSE_", "LANGFUSE_")):
called["n"] += 1
return real_get(key, default)
monkeypatch.setattr(os.environ, "get", tracking_get)
for _ in range(20):
assert langfuse_plugin._get_langfuse() is None
assert called["n"] == 0, (
f"_get_langfuse() re-read env {called['n']} times after cache miss — "
"it should short-circuit via _INIT_FAILED"
)
def test_get_langfuse_does_not_import_hermes_config(self, monkeypatch):
"""The plugin must not re-read config.yaml per hook."""
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
# Drop any cached import of hermes_cli.config.
sys.modules.pop("hermes_cli.config", None)
langfuse_plugin = self._fresh_plugin()
for _ in range(20):
langfuse_plugin._get_langfuse()
assert "hermes_cli.config" not in sys.modules, (
"langfuse plugin imported hermes_cli.config — regression toward "
"the rejected per-hook load_config() design"
)
# ---------------------------------------------------------------------------
# Hooks are inert when the client is unavailable.
# ---------------------------------------------------------------------------
class TestHooksInert:
def test_hooks_noop_without_client(self, monkeypatch):
"""All 6 hooks must return without raising when _get_langfuse() is None."""
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
sys.modules.pop("plugins.observability.langfuse", None)
import importlib
mod = importlib.import_module("plugins.observability.langfuse")
# Each hook should just return; no exceptions.
mod.on_pre_llm_call(task_id="t", session_id="s", messages=[{"role": "user", "content": "hi"}])
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, request_messages=[])
mod.on_post_llm_call(task_id="t", session_id="s", api_call_count=1)
mod.on_pre_tool_call(tool_name="read_file", args={}, task_id="t", session_id="s")
mod.on_post_tool_call(tool_name="read_file", args={}, result="ok", task_id="t", session_id="s")
# ---------------------------------------------------------------------------
# Placeholder-credential guard (#23823).
#
# Regression coverage for the silent-failure bug: when an operator leaves
# HERMES_LANGFUSE_PUBLIC_KEY / SECRET_KEY at a template value like
# "placeholder", "test-key", or "your-langfuse-key", the SDK accepts the
# credentials at construction time (it does no server-side validation
# eagerly) but drops every trace at flush time, with no signal in the
# Hermes logs. The fix in `_get_langfuse()` validates the documented
# `pk-lf-` / `sk-lf-` prefix Langfuse always issues, surfaces a one-shot
# warning naming the offending env var(s), and short-circuits via the
# same `_INIT_FAILED` path used for missing credentials so subsequent
# hook invocations don't re-log.
# ---------------------------------------------------------------------------
class _FakeLangfuse:
"""Stand-in for the real :class:`langfuse.Langfuse` so tests don't
need the optional ``langfuse`` SDK installed. The plugin's runtime
gate refuses to proceed past ``if Langfuse is None`` when the SDK
is missing, which would short-circuit before the placeholder check
can fire. Patching ``plugin.Langfuse`` with this class lets the
placeholder validator exercise its full code path."""
instances: list["_FakeLangfuse"] = []
def __init__(self, **kwargs):
self.kwargs = kwargs
_FakeLangfuse.instances.append(self)
class TestPlaceholderKeyDetection:
LOGGER_NAME = "plugins.observability.langfuse"
def _fresh_plugin(self, monkeypatch=None):
mod_name = "plugins.observability.langfuse"
sys.modules.pop(mod_name, None)
mod = importlib.import_module(mod_name)
if monkeypatch is not None:
# Pretend the SDK is installed so `_get_langfuse()` actually
# reaches the placeholder check. Real SDK calls are never
# made because the placeholder/missing-credentials paths
# return before constructing a client.
_FakeLangfuse.instances.clear()
monkeypatch.setattr(mod, "Langfuse", _FakeLangfuse, raising=False)
return mod
@staticmethod
def _clear_env(monkeypatch):
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
# -- helper unit tests (no SDK stub needed: these don't go through
# _get_langfuse, they exercise the pure-Python helpers directly) ------
def test_redact_key_preview_empty(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._redact_key_preview("") == "<empty>"
def test_redact_key_preview_short_value_echoed(self, monkeypatch):
"""Short placeholder strings are echoed in full so the operator
can see exactly which template they forgot to replace."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._redact_key_preview("placeholder") == "'placeholder'"
assert plugin._redact_key_preview("test-key") == "'test-key'"
def test_redact_key_preview_long_value_truncated(self, monkeypatch):
"""If an operator pasted a real secret into the wrong env var the
preview must NOT echo it in full — only the leading 6 chars."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
result = plugin._redact_key_preview("sk-lf-abcdefghijklmnop")
assert "abcdefghij" not in result
assert result.startswith("'sk-lf-")
assert result.endswith("...'")
def test_validate_langfuse_key_accepts_documented_prefix(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._validate_langfuse_key(
"HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz"
) is None
assert plugin._validate_langfuse_key(
"HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz"
) is None
def test_validate_langfuse_key_rejects_wrong_prefix(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
msg = plugin._validate_langfuse_key(
"HERMES_LANGFUSE_PUBLIC_KEY", "placeholder"
)
assert msg is not None
assert "HERMES_LANGFUSE_PUBLIC_KEY" in msg
assert "pk-lf-" in msg
def test_validate_langfuse_key_unknown_name_passes(self, monkeypatch):
"""Defensive: an env var with no registered prefix is trusted."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._validate_langfuse_key("HERMES_LANGFUSE_BASE_URL", "anything") is None
# -- end-to-end _get_langfuse() behaviour --------------------------------
# These tests pass `monkeypatch` to _fresh_plugin() so the helper can
# stub out `Langfuse` (the optional SDK). Without that, every call
# short-circuits at `if Langfuse is None` before reaching the
# placeholder validator — masking the very behaviour we're testing.
def test_placeholder_public_key_warns_and_skips(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
text = caplog.text
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
assert "'placeholder'" in text
assert "pk-lf-" in text
# The valid secret value must NOT appear (the var NAME does, in
# the "or unset ..." hint, but the value preview shouldn't).
assert "'sk-lf-" not in text
# Never constructed the SDK client — short-circuited before that.
assert _FakeLangfuse.instances == []
def test_placeholder_secret_key_warns_and_skips(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "test-key")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
text = caplog.text
assert "HERMES_LANGFUSE_SECRET_KEY" in text
assert "'test-key'" in text
assert "sk-lf-" in text
# The valid public value must NOT appear.
assert "'pk-lf-" not in text
assert _FakeLangfuse.instances == []
def test_both_placeholders_one_warning_with_both_keys(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert len(warnings) == 1, (
f"Expected a single combined warning; got {len(warnings)}:\n"
+ "\n".join(r.getMessage() for r in warnings)
)
text = warnings[0].getMessage()
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
assert "HERMES_LANGFUSE_SECRET_KEY" in text
def test_repeated_calls_do_not_re_warn(self, monkeypatch, caplog):
"""The cached ``_INIT_FAILED`` sentinel must short-circuit
subsequent calls so each hook invocation isn't a fresh log
line — otherwise a busy gateway will spam the operator's
terminal."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
for _ in range(15):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert len(warnings) == 1, (
f"Warning fired {len(warnings)} times across 15 calls; "
"expected 1 (cached via _INIT_FAILED)"
)
@pytest.mark.parametrize("placeholder", [
"placeholder",
"test-key",
"your-langfuse-key",
"change-me",
"xxx",
"dummy-key-here",
"<your-key>",
"REPLACE_ME",
])
def test_common_placeholders_detected(self, monkeypatch, caplog, placeholder):
"""A grab-bag of values that real-world ``.env.example`` templates
use as stand-ins. Any of them in either key must trip the guard."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", placeholder)
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
def test_legacy_LANGFUSE_PUBLIC_KEY_also_validated(self, monkeypatch, caplog):
"""The plugin reads both the canonical HERMES_-prefixed env var and
the legacy bare ``LANGFUSE_PUBLIC_KEY``. The validator must run on
whichever value ``_get_langfuse()`` actually consumed."""
self._clear_env(monkeypatch)
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
# Warning names the canonical user-facing env var (the bare
# LANGFUSE_PUBLIC_KEY is a backwards-compat alias for the
# HERMES_-prefixed one — operators set the HERMES_-prefixed one).
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
assert "'placeholder'" in caplog.text
def test_missing_credentials_still_skip_silently(self, monkeypatch, caplog):
"""Missing-creds is the documented opt-out path (operator hasn't
configured the plugin yet) — it must remain SILENT. Regression
guard against the placeholder validator accidentally running on
empty values and re-introducing log noise for unconfigured
installs."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert warnings == []
def test_sdk_not_installed_still_skips_silently(self, monkeypatch, caplog):
"""If the langfuse SDK isn't installed at all, the placeholder
check should never run — there's nothing the operator can do
about a credential mismatch when the package is missing, and
re-warning here would dilute the actually-actionable SDK-missing
signal upstream. The ``Langfuse is None`` guard at the top of
``_get_langfuse`` already handles this; this test pins that
behaviour."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
# NO monkeypatch on Langfuse here — falls back to whatever the
# plugin imported at module load (None if SDK absent).
plugin = self._fresh_plugin()
monkeypatch.setattr(plugin, "Langfuse", None, raising=False)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert warnings == []
def test_valid_prefixes_do_not_trigger_placeholder_warning(self, monkeypatch, caplog):
"""Real Langfuse keys (``pk-lf-…`` / ``sk-lf-…``) must pass the
guard and proceed to SDK init. We stub the SDK constructor with
a recording fake so the assertion can confirm BOTH that the
placeholder warning didn't fire AND that the client was actually
constructed — the latter is the success signal the bug report
wanted."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
client = plugin._get_langfuse()
assert isinstance(client, _FakeLangfuse)
assert client.kwargs["public_key"] == "pk-lf-real-public-xyz"
assert client.kwargs["secret_key"] == "sk-lf-real-secret-xyz"
assert "placeholders" not in caplog.text.lower(), (
f"Valid Langfuse keys tripped the placeholder guard: {caplog.text!r}"
)
class TestRequestMessageCoercion:
def test_prefers_request_messages_then_messages_then_history_then_user_message(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
assert mod._coerce_request_messages(
request_messages=[{"role": "system", "content": "s"}],
messages=[{"role": "user", "content": "m"}],
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "system", "content": "s"}]
assert mod._coerce_request_messages(
messages=[{"role": "user", "content": "m"}],
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "user", "content": "m"}]
assert mod._coerce_request_messages(
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "user", "content": "h"}]
assert mod._coerce_request_messages(user_message="u") == [{"role": "user", "content": "u"}]
class TestToolCallOutputBackfill:
def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
observation = object()
state = mod.TraceState(trace_id="trace-1", root_ctx=None, root_span=None)
state.tools["call-1"] = observation
state.turn_tool_calls.append({
"id": "call-1",
"type": "function",
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
"function": {
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
},
})
task_key = mod._trace_key("task-1", "session-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):
ended["observation"] = obs
ended["output"] = output
ended["metadata"] = metadata
monkeypatch.setattr(mod, "_end_observation", fake_end_observation)
mod.on_post_tool_call(
tool_name="web_extract",
args={"urls": ["https://example.com"]},
result='{"results": [{"url": "https://example.com", "content": "Example Domain"}]}',
task_id="task-1",
session_id="session-1",
tool_call_id="call-1",
)
assert ended["observation"] is observation
assert state.turn_tool_calls[0]["output"] == ended["output"]
assert state.turn_tool_calls[0]["function"]["output"] == ended["output"]
assert state.turn_tool_calls[0]["output"] == {
"results": [{"url": "https://example.com", "content": "Example Domain"}]
}
def test_serialize_messages_keeps_tool_name_and_call_id(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
messages = [{
"role": "tool",
"name": "web_extract",
"tool_call_id": "call-1",
"content": '{"ok": true}',
}]
assert mod._serialize_messages(messages) == [{
"role": "tool",
"name": "web_extract",
"tool_call_id": "call-1",
"content": {"ok": True},
}]
def test_serialize_tool_calls_emits_openai_style_function_shape(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
class _Fn:
name = "web_extract"
arguments = '{"urls": ["https://example.com"]}'
class _ToolCall:
id = "call-1"
type = "function"
function = _Fn()
assert mod._serialize_tool_calls([_ToolCall()]) == [{
"id": "call-1",
"type": "function",
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
"function": {
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
},
}]
class TestToolObservationKeying:
"""Tests for pre/post tool_call observation matching when tool_call_id is absent."""
def _make_mod(self):
sys.modules.pop("plugins.observability.langfuse", None)
return importlib.import_module("plugins.observability.langfuse")
def test_empty_tool_call_id_single_tool_sets_output(self, monkeypatch):
mod = self._make_mod()
obs = object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name.setdefault("my_tool", []).append(obs)
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end(o, *, output=None, metadata=None, **kw):
ended["obs"] = o
ended["output"] = output
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="my_tool",
args={},
result='{"ok": true}',
task_id="task-1",
session_id="sess-1",
tool_call_id="",
)
assert ended["obs"] is obs
assert ended["output"] == {"ok": True}
assert state.pending_tools_by_name.get("my_tool") is None
def test_empty_tool_call_id_observations_are_fifo_within_tool_name(self, monkeypatch):
"""Two queued observations are consumed in FIFO order so the first
post hook gets the first observation's output, not the second.
Sequential-on-one-thread coverage; the real concurrent case is
guarded by ``_STATE_LOCK`` around every read-modify-write on
``pending_tools_by_name`` and is exercised in
``test_threaded_post_calls_preserve_fifo_under_lock`` below.
"""
mod = self._make_mod()
obs_a, obs_b = object(), object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name["web_extract"] = [obs_a, obs_b]
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
calls = []
def fake_end(o, *, output=None, metadata=None, **kw):
calls.append((o, output))
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"val": "a"}',
task_id="task-1", session_id="sess-1", tool_call_id="",
)
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"val": "b"}',
task_id="task-1", session_id="sess-1", tool_call_id="",
)
assert calls[0] == (obs_a, {"val": "a"})
assert calls[1] == (obs_b, {"val": "b"})
assert state.pending_tools_by_name.get("web_extract") is None
def test_threaded_post_calls_preserve_fifo_under_lock(self, monkeypatch):
"""The actual concurrency contract: when 8 threads race to drain
the pending queue, no observation is consumed twice and none is
lost. Validates ``_STATE_LOCK`` discipline, not Python list
semantics."""
import threading
mod = self._make_mod()
n = 8
observations = [object() for _ in range(n)]
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name["web_extract"] = list(observations)
task_key = mod._trace_key("task-thr", "sess-thr")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
recorded: list = []
lock = threading.Lock()
def fake_end(o, *, output=None, metadata=None, **kw):
with lock:
recorded.append(o)
monkeypatch.setattr(mod, "_end_observation", fake_end)
barrier = threading.Barrier(n)
def worker():
barrier.wait()
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"ok": true}',
task_id="task-thr", session_id="sess-thr", tool_call_id="",
)
threads = [threading.Thread(target=worker) for _ in range(n)]
for t in threads:
t.start()
for t in threads:
t.join()
# Every observation was consumed exactly once; queue is empty.
assert len(recorded) == n
assert set(map(id, recorded)) == set(map(id, observations))
assert state.pending_tools_by_name.get("web_extract") is None
def test_explicit_tool_call_id_uses_tools_dict(self, monkeypatch):
"""When tool_call_id is present, pending_tools_by_name is not touched."""
mod = self._make_mod()
obs = object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.tools["call-99"] = obs
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end(o, *, output=None, metadata=None, **kw):
ended["obs"] = o
ended["output"] = output
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="my_tool", args={}, result='{"status": "done"}',
task_id="task-1", session_id="sess-1", tool_call_id="call-99",
)
assert ended["obs"] is obs
assert ended["output"] == {"status": "done"}
assert not state.tools
+738
View File
@@ -0,0 +1,738 @@
"""Tests for the RetainDB memory plugin.
Covers: _Client HTTP client, _WriteQueue SQLite queue, _build_overlay formatter,
RetainDBMemoryProvider lifecycle/tools/prefetch, thread management, connection pooling.
"""
import json
import sqlite3
import time
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Imports — guarded since plugins/memory lives outside the standard test path
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(tmp_path, monkeypatch):
"""Ensure HERMES_HOME and RETAINDB vars are isolated."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.delenv("RETAINDB_API_KEY", raising=False)
monkeypatch.delenv("RETAINDB_BASE_URL", raising=False)
monkeypatch.delenv("RETAINDB_PROJECT", raising=False)
@pytest.fixture(autouse=True)
def _cap_retaindb_sleeps(monkeypatch):
"""Cap production-code sleeps so background-thread tests run fast.
The retaindb ``_WriteQueue._flush_row`` does ``time.sleep(2)`` after
errors. Across multiple tests that trigger the retry path, that adds
up. Cap the module's bound ``time.sleep`` to 0.05s — tests don't care
about the exact retry delay, only that it happens. The test file's
own ``time.sleep`` stays real since it uses a different reference.
"""
try:
from plugins.memory import retaindb as _retaindb
except ImportError:
return
real_sleep = _retaindb.time.sleep
def _capped_sleep(seconds):
return real_sleep(min(float(seconds), 0.05))
import types as _types
fake_time = _types.SimpleNamespace(sleep=_capped_sleep, time=_retaindb.time.time)
monkeypatch.setattr(_retaindb, "time", fake_time)
# We need the repo root on sys.path so the plugin can import agent.memory_provider
import sys
_repo_root = str(Path(__file__).resolve().parents[2])
if _repo_root not in sys.path:
sys.path.insert(0, _repo_root)
from plugins.memory.retaindb import (
_Client,
_WriteQueue,
_build_overlay,
RetainDBMemoryProvider,
)
# ===========================================================================
# _Client tests
# ===========================================================================
class TestClient:
"""Test the HTTP client with mocked requests."""
def _make_client(self, api_key="rdb-test-key", base_url="https://api.retaindb.com", project="test"):
return _Client(api_key, base_url, project)
def test_base_url_trailing_slash_stripped(self):
c = self._make_client(base_url="https://api.retaindb.com///")
assert c.base_url == "https://api.retaindb.com"
def test_headers_include_auth(self):
c = self._make_client()
h = c._headers("/v1/files")
assert h["Authorization"] == "Bearer rdb-test-key"
assert "X-API-Key" not in h
def test_headers_include_api_key_for_memory_path(self):
c = self._make_client()
h = c._headers("/v1/memory/search")
assert h["X-API-Key"] == "rdb-test-key"
def test_headers_include_api_key_for_context_path(self):
c = self._make_client()
h = c._headers("/v1/context/query")
assert h["X-API-Key"] == "rdb-test-key"
def test_headers_strip_bearer_prefix(self):
c = self._make_client(api_key="Bearer rdb-test-key")
h = c._headers("/v1/memory/search")
assert h["Authorization"] == "Bearer rdb-test-key"
assert h["X-API-Key"] == "rdb-test-key"
def test_add_memory_tries_fallback(self):
c = self._make_client()
call_count = 0
def fake_request(method, path, **kwargs):
nonlocal call_count
call_count += 1
if call_count == 1:
raise RuntimeError("404")
return {"id": "mem-1"}
with patch.object(c, "request", side_effect=fake_request):
result = c.add_memory("u1", "s1", "test fact")
assert result == {"id": "mem-1"}
assert call_count == 2
def test_delete_memory_tries_fallback(self):
c = self._make_client()
call_count = 0
def fake_request(method, path, **kwargs):
nonlocal call_count
call_count += 1
if call_count == 1:
raise RuntimeError("404")
return {"deleted": True}
with patch.object(c, "request", side_effect=fake_request):
result = c.delete_memory("mem-123")
assert result == {"deleted": True}
assert call_count == 2
# ===========================================================================
# _WriteQueue tests
# ===========================================================================
class TestWriteQueue:
"""Test the SQLite-backed write queue with real SQLite."""
def _make_queue(self, tmp_path, client=None):
if client is None:
client = MagicMock()
client.ingest_session = MagicMock(return_value={"status": "ok"})
db_path = tmp_path / "test_queue.db"
return _WriteQueue(client, db_path), client, db_path
def test_enqueue_creates_row(self, tmp_path):
q, client, db_path = self._make_queue(tmp_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
# shutdown() blocks until the writer thread drains the queue — no need
# to pre-sleep (the old 1s sleep was a just-in-case wait, but shutdown
# does the right thing).
q.shutdown()
# If ingest succeeded, the row should be deleted
client.ingest_session.assert_called_once()
def test_enqueue_persists_to_sqlite(self, tmp_path):
client = MagicMock()
# Make ingest slow so the row is still in SQLite when we peek.
# 0.5s is plenty — the test just needs the flush to still be in-flight.
client.ingest_session = MagicMock(side_effect=lambda *a, **kw: time.sleep(0.5))
db_path = tmp_path / "test_queue.db"
q = _WriteQueue(client, db_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "test"}])
# Check SQLite directly — row should exist since flush is slow
conn = sqlite3.connect(str(db_path))
rows = conn.execute("SELECT user_id, session_id FROM pending").fetchall()
conn.close()
assert len(rows) >= 1
assert rows[0][0] == "user1"
q.shutdown()
def test_flush_deletes_row_on_success(self, tmp_path):
q, client, db_path = self._make_queue(tmp_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
q.shutdown() # blocks until drain
# Row should be gone
conn = sqlite3.connect(str(db_path))
rows = conn.execute("SELECT COUNT(*) FROM pending").fetchone()[0]
conn.close()
assert rows == 0
def test_flush_records_error_on_failure(self, tmp_path):
client = MagicMock()
client.ingest_session = MagicMock(side_effect=RuntimeError("API down"))
db_path = tmp_path / "test_queue.db"
q = _WriteQueue(client, db_path)
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
# Poll for the error to be recorded (max 2s), instead of a fixed 3s wait.
deadline = time.time() + 2.0
last_error = None
while time.time() < deadline:
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT last_error FROM pending").fetchone()
conn.close()
if row and row[0]:
last_error = row[0]
break
time.sleep(0.05)
q.shutdown()
assert last_error is not None
assert "API down" in last_error
def test_thread_local_connection_reuse(self, tmp_path):
q, _, _ = self._make_queue(tmp_path)
# Same thread should get same connection
conn1 = q._get_conn()
conn2 = q._get_conn()
assert conn1 is conn2
q.shutdown()
def test_crash_recovery_replays_pending(self, tmp_path):
"""Simulate crash: create rows, then new queue should replay them."""
db_path = tmp_path / "recovery_test.db"
# First: create a queue and insert rows, but don't let them flush
client1 = MagicMock()
client1.ingest_session = MagicMock(side_effect=RuntimeError("fail"))
q1 = _WriteQueue(client1, db_path)
q1.enqueue("user1", "sess1", [{"role": "user", "content": "lost turn"}])
# Wait until the error is recorded (poll with short interval).
deadline = time.time() + 2.0
while time.time() < deadline:
conn = sqlite3.connect(str(db_path))
row = conn.execute("SELECT last_error FROM pending").fetchone()
conn.close()
if row and row[0]:
break
time.sleep(0.05)
q1.shutdown()
# Now create a new queue — it should replay the pending rows
client2 = MagicMock()
client2.ingest_session = MagicMock(return_value={"status": "ok"})
q2 = _WriteQueue(client2, db_path)
# Poll for the replay to happen.
deadline = time.time() + 2.0
while time.time() < deadline:
if client2.ingest_session.called:
break
time.sleep(0.05)
q2.shutdown()
# The replayed row should have been ingested via client2
client2.ingest_session.assert_called_once()
call_args = client2.ingest_session.call_args
assert call_args[0][0] == "user1" # user_id
# ===========================================================================
# _build_overlay tests
# ===========================================================================
class TestBuildOverlay:
"""Test the overlay formatter (pure function)."""
def test_empty_inputs_returns_empty(self):
assert _build_overlay({}, {}) == ""
def test_empty_memories_returns_empty(self):
assert _build_overlay({"memories": []}, {"results": []}) == ""
def test_profile_items_included(self):
profile = {"memories": [{"content": "User likes Python"}]}
result = _build_overlay(profile, {})
assert "User likes Python" in result
assert "[RetainDB Context]" in result
def test_query_results_included(self):
query_result = {"results": [{"content": "Previous discussion about Rust"}]}
result = _build_overlay({}, query_result)
assert "Previous discussion about Rust" in result
def test_deduplication_removes_duplicates(self):
profile = {"memories": [{"content": "User likes Python"}]}
query_result = {"results": [{"content": "User likes Python"}]}
result = _build_overlay(profile, query_result)
assert result.count("User likes Python") == 1
def test_local_entries_filter(self):
profile = {"memories": [{"content": "Already known fact"}]}
result = _build_overlay(profile, {}, local_entries=["Already known fact"])
# The profile item matches a local entry, should be filtered
assert result == ""
def test_max_five_items_per_section(self):
profile = {"memories": [{"content": f"Fact {i}"} for i in range(10)]}
result = _build_overlay(profile, {})
# Should only include first 5
assert "Fact 0" in result
assert "Fact 4" in result
assert "Fact 5" not in result
def test_none_content_handled(self):
profile = {"memories": [{"content": None}, {"content": "Real fact"}]}
result = _build_overlay(profile, {})
assert "Real fact" in result
def test_truncation_at_320_chars(self):
long_content = "x" * 500
profile = {"memories": [{"content": long_content}]}
result = _build_overlay(profile, {})
# Each item is compacted to 320 chars max
for line in result.split("\n"):
if line.startswith("- "):
assert len(line) <= 322 # "- " + 320
# ===========================================================================
# RetainDBMemoryProvider tests
# ===========================================================================
class TestRetainDBMemoryProvider:
"""Test the main plugin class."""
def _make_provider(self, tmp_path, monkeypatch, api_key="rdb-test-key"):
monkeypatch.setenv("RETAINDB_API_KEY", api_key)
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
(tmp_path / ".hermes").mkdir(exist_ok=True)
provider = RetainDBMemoryProvider()
return provider
def test_name(self):
p = RetainDBMemoryProvider()
assert p.name == "retaindb"
def test_is_available_without_key(self):
p = RetainDBMemoryProvider()
assert p.is_available() is False
def test_is_available_with_key(self, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test")
p = RetainDBMemoryProvider()
assert p.is_available() is True
def test_config_schema(self):
p = RetainDBMemoryProvider()
schema = p.get_config_schema()
assert len(schema) == 3
keys = [s["key"] for s in schema]
assert "api_key" in keys
assert "base_url" in keys
assert "project" in keys
def test_initialize_creates_client_and_queue(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
assert p._client is not None
assert p._queue is not None
assert p._session_id == "test-session"
p.shutdown()
def test_initialize_default_project(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
assert p._client.project == "default"
p.shutdown()
def test_initialize_explicit_project(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_PROJECT", "my-project")
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
assert p._client.project == "my-project"
p.shutdown()
def test_initialize_profile_project(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
profile_home = str(tmp_path / "profiles" / "coder")
p.initialize("test-session", hermes_home=profile_home)
assert p._client.project == "hermes-coder"
p.shutdown()
def test_initialize_seeds_soul_md(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
soul_path = tmp_path / ".hermes" / "SOUL.md"
soul_path.write_text("I am a helpful agent.")
with patch.object(RetainDBMemoryProvider, "_seed_soul") as mock_seed:
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
# Give thread time to start
time.sleep(0.5)
mock_seed.assert_called_once_with("I am a helpful agent.")
p.shutdown()
def test_system_prompt_block(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
block = p.system_prompt_block()
assert "RetainDB Memory" in block
assert "Active" in block
p.shutdown()
def test_handle_tool_call_not_initialized(self):
p = RetainDBMemoryProvider()
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
assert "error" in result
assert "not initialized" in result["error"]
def test_handle_tool_call_unknown_tool(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_nonexistent", {}))
assert result == {"error": "Unknown tool: retaindb_nonexistent"}
p.shutdown()
def test_dispatch_profile(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "get_profile", return_value={"memories": []}):
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
assert "memories" in result
p.shutdown()
def test_dispatch_search_requires_query(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_search", {}))
assert result == {"error": "query is required"}
p.shutdown()
def test_dispatch_search(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "search", return_value={"results": [{"content": "found"}]}):
result = json.loads(p.handle_tool_call("retaindb_search", {"query": "test"}))
assert "results" in result
p.shutdown()
def test_dispatch_search_top_k_capped(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "search") as mock_search:
mock_search.return_value = {"results": []}
p.handle_tool_call("retaindb_search", {"query": "test", "top_k": 100})
# top_k should be capped at 20
assert mock_search.call_args[1]["top_k"] == 20
p.shutdown()
def test_dispatch_remember(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}):
result = json.loads(p.handle_tool_call("retaindb_remember", {"content": "test fact"}))
assert result["id"] == "mem-1"
p.shutdown()
def test_dispatch_remember_requires_content(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_remember", {}))
assert result == {"error": "content is required"}
p.shutdown()
def test_dispatch_forget(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "delete_memory", return_value={"deleted": True}):
result = json.loads(p.handle_tool_call("retaindb_forget", {"memory_id": "mem-1"}))
assert result["deleted"] is True
p.shutdown()
def test_dispatch_forget_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_forget", {}))
assert result == {"error": "memory_id is required"}
p.shutdown()
def test_dispatch_context(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "query_context", return_value={"results": [{"content": "relevant"}]}), \
patch.object(p._client, "get_profile", return_value={"memories": []}):
result = json.loads(p.handle_tool_call("retaindb_context", {"query": "current task"}))
assert "context" in result
assert "raw" in result
p.shutdown()
def test_dispatch_file_list(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "list_files", return_value={"files": []}):
result = json.loads(p.handle_tool_call("retaindb_list_files", {}))
assert "files" in result
p.shutdown()
def test_dispatch_file_upload_missing_path(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_upload_file", {}))
assert "error" in result
def test_dispatch_file_upload_not_found(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_upload_file", {"local_path": "/nonexistent/file.txt"}))
assert "File not found" in result["error"]
p.shutdown()
def test_dispatch_file_read_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_read_file", {}))
assert result == {"error": "file_id is required"}
p.shutdown()
def test_dispatch_file_ingest_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_ingest_file", {}))
assert result == {"error": "file_id is required"}
p.shutdown()
def test_dispatch_file_delete_requires_id(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
result = json.loads(p.handle_tool_call("retaindb_delete_file", {}))
assert result == {"error": "file_id is required"}
p.shutdown()
def test_handle_tool_call_wraps_exception(self, tmp_path, monkeypatch):
p = self._make_provider(tmp_path, monkeypatch)
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
with patch.object(p._client, "get_profile", side_effect=RuntimeError("API exploded")):
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
assert "API exploded" in result["error"]
p.shutdown()
# ===========================================================================
# Prefetch and thread management tests
# ===========================================================================
class TestPrefetch:
"""Test background prefetch and thread accumulation prevention."""
def _make_initialized_provider(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
return p
def test_queue_prefetch_skips_without_client(self):
p = RetainDBMemoryProvider()
p.queue_prefetch("test") # Should not raise
def test_prefetch_returns_empty_when_nothing_cached(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
result = p.prefetch("test")
assert result == ""
p.shutdown()
def test_prefetch_consumes_context_result(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
# Manually set the cached result
with p._lock:
p._context_result = "[RetainDB Context]\nProfile:\n- User likes tests"
result = p.prefetch("test")
assert "User likes tests" in result
# Should be consumed
assert p.prefetch("test") == ""
p.shutdown()
def test_prefetch_consumes_dialectic_result(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
with p._lock:
p._dialectic_result = "User is a software engineer who prefers Python."
result = p.prefetch("test")
assert "[RetainDB User Synthesis]" in result
assert "software engineer" in result
p.shutdown()
def test_prefetch_consumes_agent_model(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
with p._lock:
p._agent_model = {
"memory_count": 5,
"persona": "Helpful coding assistant",
"persistent_instructions": ["Be concise", "Use Python"],
"working_style": "Direct and efficient",
}
result = p.prefetch("test")
assert "[RetainDB Agent Self-Model]" in result
assert "Helpful coding assistant" in result
assert "Be concise" in result
assert "Direct and efficient" in result
p.shutdown()
def test_prefetch_skips_empty_agent_model(self, tmp_path, monkeypatch):
p = self._make_initialized_provider(tmp_path, monkeypatch)
with p._lock:
p._agent_model = {"memory_count": 0}
result = p.prefetch("test")
assert "Agent Self-Model" not in result
p.shutdown()
def test_thread_accumulation_guard(self, tmp_path, monkeypatch):
"""Verify old prefetch threads are joined before new ones spawn."""
p = self._make_initialized_provider(tmp_path, monkeypatch)
# Mock the prefetch methods to be slow
with patch.object(p, "_prefetch_context", side_effect=lambda q: time.sleep(0.5)), \
patch.object(p, "_prefetch_dialectic", side_effect=lambda q: time.sleep(0.5)), \
patch.object(p, "_prefetch_agent_model", side_effect=lambda: time.sleep(0.5)):
p.queue_prefetch("query 1")
first_threads = list(p._prefetch_threads)
assert len(first_threads) == 3
# Call again — should join first batch before spawning new
p.queue_prefetch("query 2")
second_threads = list(p._prefetch_threads)
assert len(second_threads) == 3
# Should be different thread objects
for t in second_threads:
assert t not in first_threads
p.shutdown()
def test_reasoning_level_short(self):
assert RetainDBMemoryProvider._reasoning_level("hi") == "low"
def test_reasoning_level_medium(self):
assert RetainDBMemoryProvider._reasoning_level("x" * 200) == "medium"
def test_reasoning_level_long(self):
assert RetainDBMemoryProvider._reasoning_level("x" * 500) == "high"
# ===========================================================================
# sync_turn tests
# ===========================================================================
class TestSyncTurn:
"""Test turn synchronization via the write queue."""
def test_sync_turn_enqueues(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._queue, "enqueue") as mock_enqueue:
p.sync_turn("user msg", "assistant msg")
mock_enqueue.assert_called_once()
args = mock_enqueue.call_args[0]
assert args[0] == "default" # user_id
assert args[1] == "test-session" # session_id
msgs = args[2]
assert len(msgs) == 2
assert msgs[0]["role"] == "user"
assert msgs[1]["role"] == "assistant"
p.shutdown()
def test_sync_turn_skips_empty_user_content(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._queue, "enqueue") as mock_enqueue:
p.sync_turn("", "assistant msg")
mock_enqueue.assert_not_called()
p.shutdown()
# ===========================================================================
# on_memory_write hook tests
# ===========================================================================
class TestOnMemoryWrite:
"""Test the built-in memory mirror hook."""
def test_mirrors_add_action(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}) as mock_add:
p.on_memory_write("add", "user", "User prefers dark mode")
mock_add.assert_called_once()
assert mock_add.call_args[1]["memory_type"] == "preference"
p.shutdown()
def test_skips_non_add_action(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory") as mock_add:
p.on_memory_write("remove", "user", "something")
mock_add.assert_not_called()
p.shutdown()
def test_skips_empty_content(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory") as mock_add:
p.on_memory_write("add", "user", "")
mock_add.assert_not_called()
p.shutdown()
def test_memory_target_maps_to_type(self, tmp_path, monkeypatch):
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir(exist_ok=True)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
p = RetainDBMemoryProvider()
p.initialize("test-session", hermes_home=str(hermes_home))
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}) as mock_add:
p.on_memory_write("add", "memory", "Some env fact")
assert mock_add.call_args[1]["memory_type"] == "factual"
p.shutdown()
# ===========================================================================
# register() test
# ===========================================================================
class TestRegister:
def test_register_calls_register_memory_provider(self):
from plugins.memory.retaindb import register
ctx = MagicMock()
register(ctx)
ctx.register_memory_provider.assert_called_once()
arg = ctx.register_memory_provider.call_args[0][0]
assert isinstance(arg, RetainDBMemoryProvider)
@@ -0,0 +1,332 @@
"""Tests for the security-guidance plugin.
Covers ``plugins/security-guidance/``:
* ``patterns.py`` data integrity every rule has a ``RuleId``, the
fail-loud import assertion is wired.
* ``_scan_content`` true positives (pickle.load, yaml.load, eval,
dangerouslySetInnerHTML, GitHub Actions workflow), true negatives
(.md skips Python rules, ``model.eval()`` doesn't trip eval),
path-only rules (``path_check``), content-only rules
(``path_filter``).
* Hooks ``transform_tool_result`` appends a warning block in warn
mode and stays out of error results; ``pre_tool_call`` blocks
writes when ``SECURITY_GUIDANCE_BLOCK=1`` and stays silent
otherwise.
* Bundled-plugin discovery via ``PluginManager.discover_and_load``.
"""
import importlib.util
import sys
import types
from pathlib import Path
import pytest
@pytest.fixture(autouse=True)
def _isolate_env(tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.delenv("SECURITY_GUIDANCE_BLOCK", raising=False)
monkeypatch.delenv("SECURITY_GUIDANCE_DISABLE", raising=False)
yield hermes_home
# ---------------------------------------------------------------------------
# Module loading
# ---------------------------------------------------------------------------
def _repo_root() -> Path:
return Path(__file__).resolve().parents[2]
def _load_patterns():
"""Import patterns.py in isolation (no plugin glue)."""
pat_path = _repo_root() / "plugins" / "security-guidance" / "patterns.py"
spec = importlib.util.spec_from_file_location(
"security_guidance_patterns_under_test", pat_path
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _load_plugin_init():
"""Import the plugin __init__.py with patterns.py as a sibling."""
plugin_dir = _repo_root() / "plugins" / "security-guidance"
if "hermes_plugins" not in sys.modules:
ns = types.ModuleType("hermes_plugins")
ns.__path__ = []
sys.modules["hermes_plugins"] = ns
spec = importlib.util.spec_from_file_location(
"hermes_plugins.security_guidance",
plugin_dir / "__init__.py",
submodule_search_locations=[str(plugin_dir)],
)
mod = importlib.util.module_from_spec(spec)
mod.__package__ = "hermes_plugins.security_guidance"
mod.__path__ = [str(plugin_dir)]
sys.modules["hermes_plugins.security_guidance"] = mod
spec.loader.exec_module(mod)
return mod
# ---------------------------------------------------------------------------
# patterns.py data integrity
# ---------------------------------------------------------------------------
class TestPatternsData:
def test_has_at_least_one_rule(self):
p = _load_patterns()
assert len(p.SECURITY_PATTERNS) >= 1
def test_every_rule_has_required_fields(self):
p = _load_patterns()
for rule in p.SECURITY_PATTERNS:
assert "ruleName" in rule
assert "reminder" in rule and rule["reminder"]
# At least one of substrings/regex/path_check must be present —
# otherwise the rule could never fire.
assert any(k in rule for k in ("substrings", "regex", "path_check")), rule
def test_rule_names_are_unique(self):
p = _load_patterns()
names = [r["ruleName"] for r in p.SECURITY_PATTERNS]
assert len(names) == len(set(names))
def test_rule_id_enum_in_sync(self):
# The upstream patterns.py asserts this at import time. If the
# set diverges, the import itself raises and this test fails.
p = _load_patterns()
rule_names = {r["ruleName"] for r in p.SECURITY_PATTERNS}
enum_names = set(p._RULE_NAME_TO_ID)
assert rule_names == enum_names
def test_rule_names_to_mask_packs_bits(self):
p = _load_patterns()
# PICKLE_DESERIALIZATION = 8, EVAL_INJECTION = 4 → bits 8 and 4 set.
mask = p.rule_names_to_mask({"pickle_deserialization", "eval_injection"})
assert mask & (1 << p.RuleId.PICKLE_DESERIALIZATION)
assert mask & (1 << p.RuleId.EVAL_INJECTION)
# ---------------------------------------------------------------------------
# _scan_content
# ---------------------------------------------------------------------------
class TestScanContent:
def test_pickle_load_in_py_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.py", "import pickle\nx = pickle.load(open('p.pkl', 'rb'))\n"
)
names = [n for n, _ in findings]
assert "pickle_deserialization" in names
def test_pickle_load_in_md_skipped_by_path_filter(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.md", "import pickle\nx = pickle.load(open('p.pkl', 'rb'))\n"
)
assert findings == []
def test_method_call_eval_does_not_trip(self):
"""model.eval() / redis.eval() / spec.eval() must not match eval_injection."""
mod = _load_plugin_init()
findings = mod._scan_content("/tmp/foo.py", "model.eval()\nout = model(x)\n")
assert "eval_injection" not in [n for n, _ in findings]
def test_bare_eval_in_py_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content("/tmp/foo.py", "result = eval(user_input)\n")
assert "eval_injection" in [n for n, _ in findings]
def test_subprocess_shell_true_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.py", "subprocess.run('ls ' + path, shell=True)\n"
)
assert "python_subprocess_shell" in [n for n, _ in findings]
def test_dangerously_set_inner_html_warns(self):
mod = _load_plugin_init()
findings = mod._scan_content(
"/tmp/foo.tsx", "<div dangerouslySetInnerHTML={{__html: x}} />"
)
assert "react_dangerously_set_html" in [n for n, _ in findings]
def test_github_workflow_path_check_fires_on_path_alone(self):
"""github_actions_workflow has no regex/substring — fires on path."""
mod = _load_plugin_init()
findings = mod._scan_content(
".github/workflows/test.yml", "name: CI\non: pull_request"
)
assert "github_actions_workflow" in [n for n, _ in findings]
def test_non_workflow_path_doesnt_trip_workflow_rule(self):
mod = _load_plugin_init()
findings = mod._scan_content("/tmp/foo.py", "name: CI")
assert "github_actions_workflow" not in [n for n, _ in findings]
def test_empty_content_returns_no_findings(self):
mod = _load_plugin_init()
assert mod._scan_content("/tmp/foo.py", "") == []
def test_huge_content_skipped(self):
mod = _load_plugin_init()
# 1 MB of content with a dangerous pattern at the end — scanner caps
# out at _MAX_SCAN_BYTES (256 KB), so this should return [].
big = "x" * (1024 * 1024) + "\npickle.load(open('p.pkl', 'rb'))\n"
assert mod._scan_content("/tmp/foo.py", big) == []
# ---------------------------------------------------------------------------
# Hooks
# ---------------------------------------------------------------------------
class TestTransformToolResultHook:
def test_warns_on_write_file_with_dangerous_content(self):
mod = _load_plugin_init()
args = {
"path": "/tmp/foo.py",
"content": "import pickle\nx = pickle.loads(b)\n",
}
result = mod._on_transform_tool_result(
tool_name="write_file",
args=args,
result='{"success": true, "bytes_written": 30}',
)
assert isinstance(result, str)
assert "Security guidance" in result
assert "pickle_deserialization" in result
# The original JSON should still be there at the start of the string.
assert result.startswith('{"success": true')
def test_no_warn_on_clean_content(self):
mod = _load_plugin_init()
args = {"path": "/tmp/foo.py", "content": "import json\nx = json.loads(b)\n"}
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"success": true}'
)
is None
)
def test_no_warn_when_result_is_error(self):
mod = _load_plugin_init()
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
# When the tool itself errored, we don't pile a security warning on
# top — the model has bigger problems to solve.
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"error": "boom"}'
)
is None
)
def test_patch_tool_new_string_scanned(self):
mod = _load_plugin_init()
args = {
"path": "/tmp/foo.py",
"old_string": "x = 1",
"new_string": "x = eval(user_input)",
}
result = mod._on_transform_tool_result(
tool_name="patch", args=args, result='{"success": true}'
)
assert isinstance(result, str)
assert "eval_injection" in result
def test_untargeted_tool_skipped(self):
mod = _load_plugin_init()
# The plugin only scans write_file/patch/skill_manage. terminal output
# should pass through untouched.
args = {"command": "echo pickle.load"}
assert (
mod._on_transform_tool_result(
tool_name="terminal", args=args, result='{"output": "pickle.load"}'
)
is None
)
def test_disable_kill_switch(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_DISABLE", "1")
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"ok": true}'
)
is None
)
def test_block_mode_makes_transform_hook_quiet(self, monkeypatch):
"""In block mode, pre_tool_call handles the warning; the transform
hook stays silent so we don't double-emit."""
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
assert (
mod._on_transform_tool_result(
tool_name="write_file", args=args, result='{"ok": true}'
)
is None
)
class TestPreToolCallHook:
def test_no_block_in_warn_mode(self):
mod = _load_plugin_init()
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
assert mod._on_pre_tool_call(tool_name="write_file", args=args) is None
def test_blocks_in_block_mode_on_dangerous_pattern(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
out = mod._on_pre_tool_call(tool_name="write_file", args=args)
assert isinstance(out, dict)
assert out["action"] == "block"
assert "pickle_deserialization" in out["message"]
assert "SECURITY_GUIDANCE_BLOCK" in out["message"] # tells user how to disable
def test_no_block_in_block_mode_on_clean_content(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"path": "/tmp/foo.py", "content": "import json\n"}
assert mod._on_pre_tool_call(tool_name="write_file", args=args) is None
def test_untargeted_tool_skipped(self, monkeypatch):
mod = _load_plugin_init()
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
args = {"command": "echo pickle.load(f)"}
assert mod._on_pre_tool_call(tool_name="terminal", args=args) is None
# ---------------------------------------------------------------------------
# Bundled-plugin discovery
# ---------------------------------------------------------------------------
class TestPluginDiscovery:
def test_loads_via_plugin_manager(self, _isolate_env, monkeypatch):
"""End-to-end: enable in config.yaml and verify the PluginManager
picks it up via the standard discovery path."""
import yaml
config = {"plugins": {"enabled": ["security-guidance"]}}
(_isolate_env / "config.yaml").write_text(yaml.safe_dump(config))
# Wipe any cached plugin state from earlier tests in this worker.
for k in list(sys.modules):
if k.startswith(("hermes_plugins", "hermes_cli.plugins")):
del sys.modules[k]
from hermes_cli.plugins import _ensure_plugins_discovered
mgr = _ensure_plugins_discovered(force=True)
loaded = set()
if hasattr(mgr, "_plugins"):
loaded = set(mgr._plugins.keys())
assert "security-guidance" in loaded
+467
View File
@@ -0,0 +1,467 @@
"""Tests for the Teams pipeline plugin package."""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from pathlib import Path
import pytest
from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest
from gateway.config import GatewayConfig, Platform, PlatformConfig
from plugins.teams_pipeline import register
from plugins.teams_pipeline.pipeline import TeamsMeetingPipeline
from plugins.teams_pipeline.store import TeamsPipelineStore
from plugins.teams_pipeline.models import MeetingArtifact
class FakeGraphClient:
def __init__(self) -> None:
self.downloaded = False
async def _transcript_meeting_resolver(client, *, meeting_id=None, join_web_url=None, tenant_id=None):
from plugins.teams_pipeline.models import TeamsMeetingRef
return TeamsMeetingRef(
meeting_id=str(meeting_id),
tenant_id=tenant_id,
metadata={"subject": "Weekly Sync", "participants": [{"displayName": "Ada"}]},
)
async def _no_call_record(*args, **kwargs):
return None
def test_register_adds_cli_only():
mgr = PluginManager()
manifest = PluginManifest(name="teams_pipeline")
ctx = PluginContext(manifest, mgr)
register(ctx)
assert "teams-pipeline" in mgr._cli_commands
entry = mgr._cli_commands["teams-pipeline"]
assert entry["plugin"] == "teams_pipeline"
assert callable(entry["setup_fn"])
assert callable(entry["handler_fn"])
def test_runtime_config_uses_existing_teams_platform_settings():
from plugins.teams_pipeline.runtime import build_pipeline_runtime_config
gateway_config = GatewayConfig(
platforms={
Platform("teams"): PlatformConfig(
enabled=True,
extra={
"delivery_mode": "graph",
"team_id": "team-1",
"channel_id": "channel-1",
"meeting_pipeline": {
"transcript_min_chars": 120,
"notion": {"enabled": True, "database_id": "db-1"},
},
},
)
}
)
runtime_config = build_pipeline_runtime_config(gateway_config)
assert runtime_config["transcript_min_chars"] == 120
assert runtime_config["notion"]["database_id"] == "db-1"
assert runtime_config["teams_delivery"] == {
"enabled": True,
"mode": "graph",
"team_id": "team-1",
"channel_id": "channel-1",
}
def test_build_pipeline_runtime_reuses_existing_teams_adapter_surface(monkeypatch, tmp_path):
from plugins.teams_pipeline import runtime as runtime_module
class FakeWriter:
def __init__(self, platform_config=None, **kwargs) -> None:
self.platform_config = platform_config
monkeypatch.setattr(runtime_module, "build_graph_client", lambda: object())
monkeypatch.setattr(runtime_module, "resolve_teams_pipeline_store_path", lambda: tmp_path / "teams-store.json")
monkeypatch.setattr("plugins.platforms.teams.adapter.TeamsSummaryWriter", FakeWriter)
gateway = SimpleNamespace(
config=GatewayConfig(
platforms={
Platform("teams"): PlatformConfig(
enabled=True,
extra={
"delivery_mode": "incoming_webhook",
"incoming_webhook_url": "https://example.com/hook",
},
)
}
)
)
runtime = runtime_module.build_pipeline_runtime(gateway)
assert isinstance(runtime.teams_sender, FakeWriter)
assert runtime.teams_sender.platform_config is gateway.config.platforms[Platform("teams")]
@pytest.mark.anyio
async def test_bind_gateway_runtime_attaches_scheduler(monkeypatch, tmp_path):
from plugins.teams_pipeline import runtime as runtime_module
class FakeAdapter:
def __init__(self) -> None:
self.scheduler = None
def set_notification_scheduler(self, scheduler) -> None:
self.scheduler = scheduler
class FakePipeline:
def __init__(self) -> None:
self.notifications = []
async def run_notification(self, notification):
self.notifications.append(notification)
adapter = FakeAdapter()
pipeline = FakePipeline()
gateway = SimpleNamespace(
adapters={Platform.MSGRAPH_WEBHOOK: adapter},
config=GatewayConfig(platforms={}),
_teams_pipeline_runtime=None,
_teams_pipeline_runtime_error=None,
)
monkeypatch.setattr(runtime_module, "build_pipeline_runtime", lambda gateway_runner: pipeline)
bound = runtime_module.bind_gateway_runtime(gateway)
assert bound is True
assert gateway._teams_pipeline_runtime is pipeline
assert callable(adapter.scheduler)
notification = {"id": "notif-1"}
await adapter.scheduler(notification, object())
assert pipeline.notifications == [notification]
@pytest.mark.anyio
async def test_bind_gateway_runtime_drops_notifications_when_unavailable(monkeypatch):
from plugins.teams_pipeline import runtime as runtime_module
from tools.microsoft_graph_auth import MicrosoftGraphConfigError
class FakeAdapter:
def __init__(self) -> None:
self.scheduler = None
def set_notification_scheduler(self, scheduler) -> None:
self.scheduler = scheduler
adapter = FakeAdapter()
gateway = SimpleNamespace(
adapters={Platform.MSGRAPH_WEBHOOK: adapter},
config=GatewayConfig(platforms={}),
_teams_pipeline_runtime=None,
_teams_pipeline_runtime_error=None,
)
def _raise(_gateway_runner):
raise MicrosoftGraphConfigError("missing graph env")
monkeypatch.setattr(runtime_module, "build_pipeline_runtime", _raise)
bound = runtime_module.bind_gateway_runtime(gateway)
assert bound is False
assert "missing graph env" in gateway._teams_pipeline_runtime_error
assert callable(adapter.scheduler)
await adapter.scheduler({"id": "notif-2"}, object())
def test_store_persists_subscription_event_and_job_state(tmp_path):
store_path = tmp_path / "teams-store.json"
store = TeamsPipelineStore(store_path)
store.upsert_subscription(
"sub-1",
{"client_state": "abc", "resource": "communications/onlineMeetings"},
)
store.record_event_timestamp("evt-1", "2026-05-03T19:30:00Z")
store.upsert_job("job-1", {"status": "received", "event_id": "evt-1"})
store.upsert_sink_record("notion:meeting-1", {"page_id": "page-1"})
reloaded = TeamsPipelineStore(store_path)
subscription = reloaded.get_subscription("sub-1")
job = reloaded.get_job("job-1")
sink = reloaded.get_sink_record("notion:meeting-1")
assert subscription is not None
assert subscription["subscription_id"] == "sub-1"
assert subscription["client_state"] == "abc"
assert reloaded.get_event_timestamp("evt-1") == "2026-05-03T19:30:00Z"
assert job is not None
assert job["status"] == "received"
assert sink is not None
assert sink["page_id"] == "page-1"
def test_store_notification_receipts_are_idempotent(tmp_path):
store = TeamsPipelineStore(tmp_path / "teams-store.json")
notification = {
"subscriptionId": "sub-1",
"resource": "communications/onlineMeetings/meeting-1",
"changeType": "updated",
}
receipt_key = TeamsPipelineStore.build_notification_receipt_key(notification)
assert store.record_notification_receipt(receipt_key, notification) is True
assert store.record_notification_receipt(receipt_key, notification) is False
assert store.has_notification_receipt(receipt_key) is True
reloaded = TeamsPipelineStore(tmp_path / "teams-store.json")
assert reloaded.has_notification_receipt(receipt_key) is True
@pytest.mark.anyio
class TestTeamsMeetingPipeline:
async def test_transcript_first_path_persists_state_and_skips_recording(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
async def _fetch_transcript(client, meeting_ref):
return (
MeetingArtifact(artifact_type="transcript", artifact_id="tx-1", display_name="meeting.vtt"),
"Action: Send draft by Friday.\nDecision: Ship the transcript-first path.\nDetailed transcript content.",
)
async def _call_record(client, meeting_ref, *, call_record_id=None, allow_permission_errors=True):
return MeetingArtifact(
artifact_type="call_record",
artifact_id="call-1",
metadata={"metrics": {"participant_count": 4}},
)
async def _summarize(**kwargs):
return pipeline_module.TeamsMeetingSummaryPayload(
meeting_ref=kwargs["resolved_meeting"],
title="Weekly Sync",
transcript_text=kwargs["transcript_text"],
summary="Short summary",
key_decisions=["Ship the transcript-first path."],
action_items=["Send draft by Friday."],
risks=["Timeline risk."],
confidence="high",
confidence_notes="Transcript available.",
source_artifacts=kwargs["artifacts"],
)
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _fetch_transcript)
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _call_record)
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={"transcript_min_chars": 20},
summarize_fn=_summarize,
)
job = await pipeline.run_notification(
{
"id": "notif-1",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-123",
"resourceData": {"id": "meeting-123"},
}
)
assert job.status == "completed"
assert job.selected_artifact_strategy == "transcript_first"
assert job.summary_payload is not None
assert job.summary_payload.summary == "Short summary"
stored = store.get_job(job.job_id)
assert stored is not None
assert stored["status"] == "completed"
async def test_recording_fallback_uses_stt_and_updates_sink_records(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
async def _no_transcript(client, meeting_ref):
return None, None
async def _recordings(client, meeting_ref):
return [
MeetingArtifact(
artifact_type="recording",
artifact_id="rec-1",
display_name="recording.mp4",
download_url="https://files.example/recording.mp4",
)
]
async def _download(client, meeting_ref, recording, destination):
target = Path(destination)
target.write_bytes(b"video-bytes")
return {"path": str(target), "size_bytes": 11, "content_type": "video/mp4"}
async def _prepare_audio(self, recording_path):
audio_path = recording_path.with_suffix(".wav")
audio_path.write_bytes(b"audio-bytes")
return audio_path
def _transcribe(file_path, model):
return {"success": True, "transcript": "Action: Follow up with Legal.\nRisk: Budget approval pending.", "provider": "local"}
async def _summarize(**kwargs):
return pipeline_module.TeamsMeetingSummaryPayload(
meeting_ref=kwargs["resolved_meeting"],
title="Weekly Sync",
transcript_text=kwargs["transcript_text"],
summary="Fallback summary",
key_decisions=[],
action_items=["Follow up with Legal."],
risks=["Budget approval pending."],
confidence="medium",
confidence_notes="Generated from STT fallback.",
source_artifacts=kwargs["artifacts"],
)
class FakeNotionWriter:
async def write_summary(self, payload, config, existing_record=None):
return {"page_id": existing_record.get("page_id") if existing_record else "page-1", "url": "https://notion.so/page-1"}
async def _teams_sender(payload, config, existing_record=None):
return {"message_id": existing_record.get("message_id") if existing_record else "msg-1"}
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _no_transcript)
monkeypatch.setattr(pipeline_module, "list_recording_artifacts", _recordings)
monkeypatch.setattr(pipeline_module, "download_recording_artifact", _download)
monkeypatch.setattr(pipeline_module.TeamsMeetingPipeline, "_prepare_audio_path", _prepare_audio)
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _no_call_record)
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={
"notion": {"enabled": True, "database_id": "db-1"},
"teams_delivery": {"enabled": True, "channel_id": "channel-1"},
},
transcribe_fn=_transcribe,
summarize_fn=_summarize,
notion_writer=FakeNotionWriter(),
teams_sender=_teams_sender,
)
job = await pipeline.run_notification(
{
"id": "notif-2",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-456",
"resourceData": {"id": "meeting-456"},
}
)
assert job.status == "completed"
assert job.selected_artifact_strategy == "recording_stt_fallback"
assert job.summary_payload is not None
assert job.summary_payload.summary == "Fallback summary"
notion_record = store.get_sink_record("notion:meeting-456")
teams_record = store.get_sink_record("teams:meeting-456")
assert notion_record is not None
assert notion_record["page_id"] == "page-1"
assert teams_record is not None
assert teams_record["message_id"] == "msg-1"
async def test_missing_transcript_and_recording_schedules_retry(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", lambda *a, **kw: asyncio.sleep(0, result=(None, None)))
monkeypatch.setattr(pipeline_module, "list_recording_artifacts", lambda *a, **kw: asyncio.sleep(0, result=[]))
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={},
summarize_fn=lambda **kwargs: asyncio.sleep(0, result=None),
)
job = await pipeline.run_notification(
{
"id": "notif-3",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-789",
"resourceData": {"id": "meeting-789"},
}
)
assert job.status == "retry_scheduled"
assert job.error_info["retryable"] is True
assert "Recording unavailable" in job.error_info["message"]
async def test_duplicate_notification_reuses_completed_job(self, tmp_path, monkeypatch):
from plugins.teams_pipeline import pipeline as pipeline_module
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
async def _fetch_transcript(client, meeting_ref):
return (
MeetingArtifact(artifact_type="transcript", artifact_id="tx-dup", display_name="meeting.vtt"),
"Decision: Keep duplicate notifications idempotent.\nAction: Verify the cached job is reused.",
)
summarize_calls = 0
async def _summarize(**kwargs):
nonlocal summarize_calls
summarize_calls += 1
return pipeline_module.TeamsMeetingSummaryPayload(
meeting_ref=kwargs["resolved_meeting"],
title="Weekly Sync",
transcript_text=kwargs["transcript_text"],
summary="Duplicate-safe summary",
key_decisions=["Keep duplicate notifications idempotent."],
action_items=["Verify the cached job is reused."],
confidence="high",
confidence_notes="Transcript available.",
source_artifacts=kwargs["artifacts"],
)
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _fetch_transcript)
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _no_call_record)
store = TeamsPipelineStore(tmp_path / "teams-store.json")
pipeline = TeamsMeetingPipeline(
graph_client=FakeGraphClient(),
store=store,
config={"transcript_min_chars": 20},
summarize_fn=_summarize,
)
notification = {
"id": "notif-dup",
"changeType": "updated",
"resource": "communications/onlineMeetings/meeting-dup",
"resourceData": {"id": "meeting-dup"},
}
first_job = await pipeline.run_notification(notification)
second_job = await pipeline.run_notification(notification)
assert first_job.status == "completed"
assert second_job.status == "completed"
assert second_job.job_id == first_job.job_id
assert summarize_calls == 1
assert len(store.list_jobs()) == 1
receipt_key = TeamsPipelineStore.build_notification_receipt_key(notification)
assert store.has_notification_receipt(receipt_key) is True
@@ -0,0 +1,431 @@
"""Behavior-parity check for the STT plugin hook + command-provider registry.
Spawns one subprocess per (version, scenario) cell pinned to either
``origin/main`` (no plugin hook, no STT command-provider registry; only
the legacy ``HERMES_LOCAL_STT_COMMAND`` escape hatch exists) or this PR's
worktree (both new surfaces present).
Each subprocess clears all STT-related env vars + writes a
``config.yaml``, then asks the dispatcher how it would route a
``transcribe_audio`` call. The emitted shape tuple is::
{dispatch_kind, provider_name, success}
Where ``dispatch_kind``
``{"builtin_local", "builtin_groq", "builtin_openai", ...,
"plugin", "plugin_unavailable", "command_provider",
"no_provider_error", "stt_disabled"}``.
Acceptable diffs:
- ``no_provider_error plugin`` for the ``plugin-installed`` scenario.
- ``no_provider_error plugin_unavailable`` for the
``plugin-installed-unavailable`` scenario (PR returns the cleaner
unavailability envelope instead of the generic auto-detect error).
- ``no_provider_error command_provider`` for the
``command-provider-installed`` scenario (registry shipped with this PR).
- ``no_provider_error command_provider`` for
``command-vs-plugin-same-name`` (command wins precedence, same as TTS).
Run from the PR worktree::
python tests/plugins/transcription/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "transcription_tools.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "transcription_tools.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "transcription_tools.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear STT-related env so dispatch decisions are config-driven.
for k in (
"GROQ_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY",
"MISTRAL_API_KEY", "XAI_API_KEY",
"HERMES_LOCAL_STT_COMMAND",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
plugin_register = sys.argv[4] # "yes" to register a fake plugin
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached from prior runs.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
# Try importing transcription_registry — only exists on PR side.
have_plugin_hook = False
try:
from agent import transcription_registry
from agent.transcription_provider import TranscriptionProvider
have_plugin_hook = True
if plugin_register == "yes":
class _FakeProvider(TranscriptionProvider):
@property
def name(self): return "openrouter"
def transcribe(self, file_path, **kw):
return {"success": True, "transcript": "PLUGIN: openrouter transcript", "provider": "openrouter"}
transcription_registry._reset_for_tests()
transcription_registry.register_provider(_FakeProvider())
elif plugin_register == "unavailable":
class _UnavailablePlugin(TranscriptionProvider):
@property
def name(self): return "openrouter"
def is_available(self): return False
def transcribe(self, file_path, **kw):
return {"success": True, "transcript": "should not run"}
transcription_registry._reset_for_tests()
transcription_registry.register_provider(_UnavailablePlugin())
except ImportError:
pass
import tools.transcription_tools as tt
# Use a real (but empty) audio file so _validate_audio_file passes.
audio_path = os.path.join(home, "audio.ogg")
with open(audio_path, "wb") as f:
# Minimal-ish OGG-shaped bytes so the size check passes.
f.write(b"OggS" + b"\x00" * 1024)
# Patch _transcribe_* so the test doesn't actually try cloud APIs.
# We're testing dispatch, not the underlying transcription.
def _stub(file_path, model_name=None):
return {"success": True, "transcript": "stub from " + sys._getframe().f_code.co_name.replace("_stub_", ""),
"provider": sys._getframe().f_code.co_name.replace("_stub_", "")}
# Stub each built-in to a marker so we can identify the branch.
class _Stub:
def __init__(self, name):
self.name = name
def __call__(self, file_path, model_name=None):
return {"success": True, "transcript": "stub", "provider": self.name}
tt._transcribe_local = _Stub("local")
tt._transcribe_local_command = _Stub("local_command")
tt._transcribe_groq = _Stub("groq")
tt._transcribe_openai = _Stub("openai")
tt._transcribe_mistral = _Stub("mistral")
tt._transcribe_xai = _Stub("xai")
# Force _get_provider to honor the explicit config since we don't have
# real creds. The provider-resolution gates check _HAS_OPENAI /
# _HAS_FASTER_WHISPER which we can't easily set, so we just patch
# _get_provider to return whatever the config says.
stt_cfg = tt._load_stt_config()
explicit = stt_cfg.get("provider")
if explicit:
# Bypass the gating for test purposes — _get_provider would
# otherwise return "none" when the dependency isn't installed.
original_get = tt._get_provider
def _patched(cfg):
if not tt.is_stt_enabled(cfg):
return "none"
return cfg.get("provider", "none")
tt._get_provider = _patched
try:
result = tt.transcribe_audio(audio_path)
except Exception as exc:
shape = {"dispatch_kind": "exception", "provider_name": None, "success": False,
"error_text": repr(exc)}
print(json.dumps(shape))
sys.exit(0)
dispatch_kind = "unknown"
provider_name = result.get("provider") if isinstance(result, dict) else None
success = result.get("success", False) if isinstance(result, dict) else False
error_text = result.get("error", "") if isinstance(result, dict) else ""
if not success and "STT is disabled" in error_text:
dispatch_kind = "stt_disabled"
elif not success and "is not available" in error_text:
dispatch_kind = "plugin_unavailable"
elif not success and "No STT provider" in error_text:
dispatch_kind = "no_provider_error"
elif provider_name in ("local", "local_command", "groq", "openai", "mistral", "xai"):
dispatch_kind = "builtin_" + provider_name
elif success and isinstance(result, dict) and result.get("transcript", "").startswith("CMD:"):
# Command-provider scenarios below emit transcripts prefixed with "CMD:"
# so the harness can distinguish command-provider dispatch from a
# plugin dispatch even when they share a provider name.
dispatch_kind = "command_provider"
elif success and isinstance(result, dict) and result.get("transcript", "").startswith("PLUGIN:"):
dispatch_kind = "plugin"
elif success and provider_name and provider_name not in ("local", "local_command", "groq", "openai", "mistral", "xai"):
dispatch_kind = "plugin"
else:
dispatch_kind = "other"
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"success": success,
}
print(json.dumps(shape))
"""
def _cmd_yaml(provider_name: str, transcript: str) -> str:
"""Build a YAML snippet for an stt.providers.<name>: type: command entry.
Produces a shell command that writes ``transcript`` to {output_path}.
Backslashes in the venv python path are doubled for YAML, and the
inner double quotes around the python -c payload are YAML-escaped.
Keeps the test scenarios readable.
"""
interp = sys.executable.replace("\\", "\\\\")
# Inside the YAML double-quoted string, we use single quotes around
# the python -c body so we don't have to YAML-escape inner double
# quotes. Single quotes inside the body are not needed; the body uses
# double quotes for module references and string literals.
payload = (
f"import sys; open(sys.argv[1], 'w').write('{transcript}')"
)
command = f'{interp} -c "{payload}" {{output_path}}'
# YAML-escape: double-quote the whole thing, escape inner " and \.
yaml_escaped = command.replace("\\", "\\\\").replace('"', '\\"')
return (
"stt:\n"
f" provider: {provider_name}\n"
" providers:\n"
f" {provider_name}:\n"
" type: command\n"
f' command: "{yaml_escaped}"\n'
)
SCENARIOS: list[tuple[str, str, dict[str, str], str]] = [
# (label, config.yaml body, scenario_env, plugin_register)
("stt-disabled", "stt:\n enabled: false\n", {}, "no"),
("explicit-groq", "stt:\n provider: groq\n", {}, "no"),
("explicit-openai", "stt:\n provider: openai\n", {}, "no"),
("explicit-local", "stt:\n provider: local\n", {}, "no"),
("explicit-xai", "stt:\n provider: xai\n", {}, "no"),
# Mistral is quarantined → _get_provider returns "none" today, hence no_provider_error.
("explicit-mistral-quarantine", "stt:\n provider: mistral\n", {}, "no"),
# Unknown name + no plugin → both: no_provider_error
("unknown-no-plugin", "stt:\n provider: openrouter\n", {}, "no"),
# Unknown name + plugin installed → main: no_provider_error, PR: plugin
("plugin-installed", "stt:\n provider: openrouter\n", {}, "yes"),
# Unknown name + plugin reports unavailable → main: no_provider_error,
# PR: plugin_unavailable (cleaner envelope, names the plugin)
("plugin-installed-unavailable", "stt:\n provider: openrouter\n", {}, "unavailable"),
# Built-in name + plugin tries to shadow → both: built-in
("explicit-openai-with-plugin-registered", "stt:\n provider: openai\n", {}, "yes"),
# NEW (this PR): stt.providers.<name>: type: command registry.
# Provider name "fake-cli" + transcript prefixed "CMD:" so dispatch_kind
# detection routes it to "command_provider". On main (no registry),
# this falls through to no_provider_error.
(
"command-provider-installed",
_cmd_yaml("fake-cli", "CMD: fake-cli transcript"),
{},
"no",
),
# NEW (this PR): same name registered as BOTH a command provider and
# a plugin under "openrouter". Command must win (config more local
# than plugin install). The plugin emits "PLUGIN:..." — assertion is
# that the transcript is "CMD:...", proving command-wins precedence.
(
"command-vs-plugin-same-name",
_cmd_yaml("openrouter", "CMD: openrouter via command wins"),
{},
"yes", # also register a plugin under "openrouter" — must NOT fire
),
# NEW (this PR): built-in name with a command provider declared under
# it → built-in still wins (built-in elif chain has precedence).
# The command would write "CMD: HIJACK" if it fired — assertion is
# that built-in OpenAI dispatch fires instead.
(
"explicit-openai-with-command-shadow",
_cmd_yaml("openai", "CMD: HIJACK"),
{},
"no",
),
]
# Subprocesses reset the registry between runs via ``_reset_for_tests`` so
# registrations from earlier scenarios don't leak. The command-provider
# scenarios also work on origin/main — the subprocess just executes the
# native dispatch path, which falls through to "no_provider_error" because
# main has no registry for stt.providers.<name>.
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict, plugin_register: str) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / "venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
plugin_register,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
return {
"dispatch_kind": shape.get("dispatch_kind"),
"success": shape.get("success"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env, plugin_register in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env, plugin_register)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env, plugin_register)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "plugin-installed" returns no_provider_error (no
# plugin hook); on PR, plugin dispatches. Same shape for
# "plugin-installed-unavailable" but PR returns the cleaner
# plugin_unavailable envelope. The new command-provider scenarios
# also intentionally diff against main (which has no stt.providers
# registry yet).
no_provider_to_plugin = (
main_reduced.get("dispatch_kind") == "no_provider_error"
and pr_reduced.get("dispatch_kind") == "plugin"
and label == "plugin-installed"
)
no_provider_to_unavailable = (
main_reduced.get("dispatch_kind") == "no_provider_error"
and pr_reduced.get("dispatch_kind") == "plugin_unavailable"
and label == "plugin-installed-unavailable"
)
no_provider_to_command = (
main_reduced.get("dispatch_kind") == "no_provider_error"
and pr_reduced.get("dispatch_kind") == "command_provider"
and label in {"command-provider-installed", "command-vs-plugin-same-name"}
)
if no_provider_to_plugin:
print(f" [DIFF] {label}: no_provider_error → plugin — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
elif no_provider_to_unavailable:
print(f" [DIFF] {label}: no_provider_error → plugin_unavailable — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
elif no_provider_to_command:
print(f" [DIFF] {label}: no_provider_error → command_provider — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"no_provider_error → plugin dispatch when a plugin is registered."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
View File
+328
View File
@@ -0,0 +1,328 @@
"""Behavior-parity check for the TTS plugin hook (issue #30398).
Spawns one subprocess per (version, scenario) cell pinned to either
``origin/main`` (no plugin hook; ``tts.provider: cartesia`` falls
through to the Edge TTS default branch) or this PR's worktree (plugin
hook present; same config routes through the plugin registry when a
plugin is registered).
Each subprocess clears all TTS-related env vars + writes a
``config.yaml``, then resolves how the dispatcher would route a
``text_to_speech`` call. The emitted shape tuple is::
{dispatch_kind, provider_name, voice_compat}
Where ``dispatch_kind``
``{"builtin_edge", "builtin_openai", "builtin_elevenlabs", ...,
"command", "plugin", "fallback_edge", "error"}``:
* ``builtin_<name>`` config selects a built-in handler that exists
on both main and PR (no diff expected)
* ``command`` config selects a ``tts.providers.<name>: type: command``
entry (PR #17843; no diff expected)
* ``plugin`` config selects a plugin-registered provider (PR only)
* ``fallback_edge`` config selects an unknown name with no matching
plugin or command entry Edge TTS default fallback
* ``error`` explicit fatal error (e.g. mistral quarantine)
The parent process diffs the reduced shape per scenario. The only
acceptable diff is ``fallback_edge plugin`` for the
``unknown-name-with-plugin-installed`` scenario everything else is
a regression.
Run from the PR worktree (it auto-resolves ``MAIN_DIR`` from the parent
of the worktree directory, or falls back to a sibling
``hermes-agent-main`` checkout)::
python tests/plugins/tts/check_parity_vs_main.py
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
def _resolve_main_dir() -> Path:
candidate = REPO_ROOT.parent.parent
if (candidate / "tools" / "tts_tool.py").exists() and candidate != REPO_ROOT:
return candidate
sibling = REPO_ROOT.parent / "hermes-agent-main"
if (sibling / "tools" / "tts_tool.py").exists():
return sibling
return REPO_ROOT
MAIN_DIR = _resolve_main_dir()
PR_DIR = REPO_ROOT
assert (PR_DIR / "tools" / "tts_tool.py").exists(), (
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
)
# The subprocess script — runs INSIDE either the main checkout or PR
# checkout, so the import paths resolve to the version of the code
# under test. We never call the real ``text_to_speech_tool`` because
# that would require audio synthesis; instead we ask the resolution
# layer what it WOULD do.
SUBPROCESS_SCRIPT = r"""
import json, os, sys, tempfile
sys.path.insert(0, sys.argv[1])
# Isolated HERMES_HOME so the config write is hermetic.
home = tempfile.mkdtemp()
os.environ["HERMES_HOME"] = home
# Clear TTS-related env so dispatch decisions are config-driven.
for k in (
"ELEVENLABS_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY",
"MINIMAX_API_KEY", "XAI_API_KEY", "GEMINI_API_KEY",
):
os.environ.pop(k, None)
scenario_env = json.loads(sys.argv[2])
os.environ.update(scenario_env)
config_yaml = sys.argv[3]
plugin_register = sys.argv[4] # "yes" to register a fake plugin
config_path = os.path.join(home, "config.yaml")
with open(config_path, "w") as f:
f.write(config_yaml)
# Fresh import — must not have anything cached from prior runs.
for name in list(sys.modules):
if (name.startswith("tools.")
or name.startswith("agent.")
or name.startswith("plugins.")
or name.startswith("hermes_cli.")):
sys.modules.pop(name, None)
# Try importing tts_registry — only exists on PR side.
have_plugin_hook = False
try:
from agent import tts_registry
from agent.tts_provider import TTSProvider
have_plugin_hook = True
if plugin_register == "yes":
class _FakeProvider(TTSProvider):
@property
def name(self): return "cartesia"
def synthesize(self, text, output_path, **kw):
return output_path
tts_registry._reset_for_tests()
tts_registry.register_provider(_FakeProvider())
except ImportError:
pass
import tools.tts_tool as tts_tool
# Read the config the same way text_to_speech_tool() does.
tts_config = tts_tool._load_tts_config()
provider = tts_tool._get_provider(tts_config)
dispatch_kind = None
provider_name = provider
voice_compat = False
error_text = None
try:
# Mistral is the one branch that returns a fatal error.
if provider == "mistral":
dispatch_kind = "error"
error_text = "mistral quarantine"
elif tts_tool._resolve_command_provider_config(provider, tts_config) is not None:
dispatch_kind = "command"
elif have_plugin_hook and provider not in tts_tool.BUILTIN_TTS_PROVIDERS:
# On PR side: check plugin dispatch.
plugin_path = tts_tool._dispatch_to_plugin_provider(
"test", os.path.join(home, "out.mp3"), provider, tts_config,
)
if plugin_path is not None:
dispatch_kind = "plugin"
voice_compat = tts_tool._plugin_provider_is_voice_compatible(provider)
else:
# Falls through to Edge TTS default on the PR side too.
dispatch_kind = "fallback_edge"
elif provider in tts_tool.BUILTIN_TTS_PROVIDERS:
dispatch_kind = "builtin_" + provider
else:
# On main side: unknown names fall through to Edge default.
dispatch_kind = "fallback_edge"
except Exception as exc:
dispatch_kind = "exception"
error_text = repr(exc)
shape = {
"dispatch_kind": dispatch_kind,
"provider_name": provider_name,
"voice_compat": bool(voice_compat),
"error_present": error_text is not None,
}
print(json.dumps(shape))
"""
SCENARIOS: list[tuple[str, str, dict[str, str], str]] = [
# (label, config.yaml body, scenario_env, plugin_register)
# Scenario 1: unset tts.provider → both: Edge default
("unset-defaults-to-edge", "", {}, "no"),
# Scenario 2: built-in name → both: that built-in
("explicit-edge", "tts:\n provider: edge\n", {}, "no"),
("explicit-openai", "tts:\n provider: openai\n", {}, "no"),
("explicit-elevenlabs", "tts:\n provider: elevenlabs\n", {}, "no"),
# Scenario 3: command-type provider → both: command dispatch
(
"command-provider",
"tts:\n provider: my-piper\n providers:\n my-piper:\n type: command\n command: 'piper -m model.onnx -f {output_path} < {input_path}'\n",
{},
"no",
),
# Scenario 4: unknown name with NO plugin installed → both: fallback to Edge
("unknown-no-plugin", "tts:\n provider: cartesia\n", {}, "no"),
# Scenario 5: unknown name WITH plugin installed
# main: fallback_edge (no plugin hook exists)
# PR: plugin (cartesia)
# This is the ONLY acceptable diff in the harness.
("plugin-installed", "tts:\n provider: cartesia\n", {}, "yes"),
# Scenario 6: built-in name + plugin tries to shadow → both: built-in
# The plugin registers under name "cartesia", not "edge", so this is
# effectively the same as scenario 2 — but we exercise the with-plugin
# path to ensure the built-in branch still takes priority.
("explicit-edge-with-plugin-registered", "tts:\n provider: edge\n", {}, "yes"),
# Scenario 7: mistral quarantine — both surface the explicit error
("mistral-quarantine", "tts:\n provider: mistral\n", {}, "no"),
]
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict, plugin_register: str) -> dict:
venv_python = repo_path / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
if not venv_python.exists():
venv_python = MAIN_DIR / "venv" / "bin" / "python"
if not venv_python.exists():
venv_python = Path("python3")
out = subprocess.run(
[
str(venv_python),
"-c",
SUBPROCESS_SCRIPT,
str(repo_path),
json.dumps(env),
config_yaml,
plugin_register,
],
capture_output=True,
text=True,
timeout=60,
)
if out.returncode != 0:
return {
"error": "subprocess failed",
"stdout": out.stdout[-500:],
"stderr": out.stderr[-500:],
}
try:
return json.loads(out.stdout.strip().splitlines()[-1])
except Exception as exc:
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
def _reduce(shape: dict) -> dict:
"""Reduce to the parts that matter for user-visible parity."""
return {
"dispatch_kind": shape.get("dispatch_kind"),
"provider_name": shape.get("provider_name"),
"error_present": shape.get("error_present"),
}
def main() -> int:
print(f"main: {MAIN_DIR}")
print(f"pr: {PR_DIR}")
print()
if MAIN_DIR == PR_DIR:
print(
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
" Set up a sibling 'hermes-agent-main' checkout pinned to "
"origin/main to get real parity coverage."
)
print()
failures: list[str] = []
errors: list[str] = []
intentional_diffs: list[tuple[str, dict, dict]] = []
for label, config_yaml, env, plugin_register in SCENARIOS:
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env, plugin_register)
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env, plugin_register)
if "error" in main_shape or "error" in pr_shape:
print(f" [ERR ] {label}: subprocess failed")
print(f" main: {main_shape}")
print(f" pr: {pr_shape}")
errors.append(label)
continue
main_reduced = _reduce(main_shape)
pr_reduced = _reduce(pr_shape)
if main_reduced == pr_reduced:
print(f" [OK] {label}: {main_reduced}")
continue
# On main, "plugin-installed" scenario returns fallback_edge
# (no plugin hook); on PR, it routes to the plugin. That's the
# only acceptable diff.
fallback_to_plugin = (
main_reduced.get("dispatch_kind") == "fallback_edge"
and pr_reduced.get("dispatch_kind") == "plugin"
and label == "plugin-installed"
)
if fallback_to_plugin:
print(f" [DIFF] {label}: fallback_edge → plugin — expected")
intentional_diffs.append((label, main_reduced, pr_reduced))
else:
print(f" [FAIL] {label}")
print(f" main: {main_reduced}")
print(f" pr: {pr_reduced}")
failures.append(label)
print()
if errors:
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
for e in errors:
print(f" - {e}")
if failures:
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
for f in failures:
print(f" - {f}")
if intentional_diffs:
print(
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
f"fallback_edge → plugin dispatch when a plugin is registered."
)
if failures or errors:
return 1
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
return 0
if __name__ == "__main__":
sys.exit(main())
+1
View File
@@ -0,0 +1 @@
"""Make tests/plugins/video_gen a package."""
+342
View File
@@ -0,0 +1,342 @@
"""Tests for the FAL video gen plugin — family routing, payload shape."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
def test_fal_provider_registers():
from plugins.video_gen.fal import FALVideoGenProvider, DEFAULT_MODEL
provider = FALVideoGenProvider()
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("fal") is provider
assert provider.display_name == "FAL"
# DEFAULT_MODEL is the cheap-tier default
assert provider.default_model() == DEFAULT_MODEL
assert DEFAULT_MODEL in {"pixverse-v6", "ltx-2.3"}
def test_fal_family_catalog():
"""Each family declares both endpoints. The catalog covers the
cheap + premium tiers Teknium listed."""
from plugins.video_gen.fal import FAL_FAMILIES
expected = {
# cheap
"ltx-2.3", "pixverse-v6",
# premium
"veo3.1", "seedance-2.0", "kling-v3-4k", "happy-horse",
}
assert expected.issubset(set(FAL_FAMILIES.keys())), (
f"missing families: {expected - set(FAL_FAMILIES.keys())}"
)
for fid, meta in FAL_FAMILIES.items():
assert meta.get("text_endpoint"), f"{fid} missing text_endpoint"
assert meta.get("image_endpoint"), f"{fid} missing image_endpoint"
assert meta["text_endpoint"] != meta["image_endpoint"]
assert meta.get("tier") in {"cheap", "premium"}, (
f"{fid} has invalid tier"
)
def test_kling_4k_uses_start_image_url():
"""Kling v3 4K's image-to-video endpoint expects start_image_url,
not image_url. The family must declare image_param_key='start_image_url'."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["kling-v3-4k"]
assert meta.get("image_param_key") == "start_image_url"
payload = _build_payload(
meta,
prompt="x",
image_url="https://example.com/i.png",
duration=5,
aspect_ratio="16:9",
resolution="720p",
negative_prompt=None,
audio=None,
seed=None,
)
assert payload.get("start_image_url") == "https://example.com/i.png"
assert "image_url" not in payload
def test_fal_list_models_advertises_both_modalities():
from plugins.video_gen.fal import FALVideoGenProvider
models = FALVideoGenProvider().list_models()
for m in models:
assert set(m["modalities"]) == {"text", "image"}, (
f"{m['id']} doesn't advertise both modalities — every family "
f"should have t2v + i2v"
)
def test_fal_unavailable_without_key(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
from plugins.video_gen import fal as fal_plugin
monkeypatch.delenv("FAL_KEY", raising=False)
# Also ensure managed gateway is unavailable
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
assert FALVideoGenProvider().is_available() is False
def test_fal_generate_requires_fal_key(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
from plugins.video_gen import fal as fal_plugin
monkeypatch.delenv("FAL_KEY", raising=False)
# Also ensure managed gateway is unavailable
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
result = FALVideoGenProvider().generate("a happy dog")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_fal_available_via_gateway(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
from plugins.video_gen import fal as fal_plugin
monkeypatch.delenv("FAL_KEY", raising=False)
monkeypatch.setattr(
fal_plugin,
"_resolve_managed_fal_video_gateway",
lambda: object(), # truthy sentinel — gateway is available
)
assert FALVideoGenProvider().is_available() is True
class TestFamilyRouting:
"""The headline behavior: image_url presence picks the endpoint."""
@pytest.fixture
def with_fake_fal(self, monkeypatch):
"""Stub fal_client.submit to capture which endpoint we hit."""
import sys
import types
captured = {"endpoint": None, "arguments": None}
class FakeHandle:
def get(self):
return {"video": {"url": "https://fake/out.mp4"}}
fake = types.ModuleType("fal_client")
def _submit(endpoint, arguments=None, headers=None):
captured["endpoint"] = endpoint
captured["arguments"] = arguments
return FakeHandle()
fake.submit = _submit # type: ignore
monkeypatch.setitem(sys.modules, "fal_client", fake)
# Reset the lazy global so it picks up our stub
from plugins.video_gen import fal as fal_plugin
fal_plugin._fal_client = None
# Also reset the managed client cache
fal_plugin._managed_fal_video_client = None
fal_plugin._managed_fal_video_client_config = None
monkeypatch.setenv("FAL_KEY", "test")
# Force direct mode — no managed gateway
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
return captured
def test_text_to_video_routes_to_text_endpoint(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"a dog running",
model="pixverse-v6",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/text-to-video"
assert result["modality"] == "text"
assert with_fake_fal["arguments"]["prompt"] == "a dog running"
assert "image_url" not in with_fake_fal["arguments"]
def test_image_to_video_routes_to_image_endpoint(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"animate this dog",
model="pixverse-v6",
image_url="https://example.com/dog.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/image-to-video"
assert result["modality"] == "image"
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
def test_default_family_text_routing(self, with_fake_fal):
"""No model arg → DEFAULT_MODEL → text-to-video endpoint."""
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate("a dog")
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_default_family_image_routing(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate(
"animate this",
image_url="https://example.com/i.png",
)
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["image_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_unknown_family_falls_back_to_default(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate(
"x",
model="not-a-real-family",
)
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_premium_seedance_routing(self, with_fake_fal):
"""Sanity check the premium-tier seedance routes correctly."""
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"a dog",
model="seedance-2.0",
image_url="https://example.com/dog.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "bytedance/seedance-2.0/image-to-video"
# Seedance uses regular image_url (not start_image_url)
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
def test_kling_4k_remaps_image_param(self, with_fake_fal):
"""Kling v3 4K image-to-video receives start_image_url, not image_url."""
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"x",
model="kling-v3-4k",
image_url="https://example.com/frame.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/kling-video/v3/4k/image-to-video"
assert with_fake_fal["arguments"].get("start_image_url") == "https://example.com/frame.png"
assert "image_url" not in with_fake_fal["arguments"]
class TestPayloadBuilder:
def test_drops_unsupported_keys(self):
"""Veo enum-clamps duration, supports aspect+resolution+audio+neg."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["veo3.1"]
p = _build_payload(
meta,
prompt="x",
image_url=None,
duration=12, # not in enum (4,6,8) — snap to 8
aspect_ratio="16:9",
resolution="720p",
negative_prompt="ugly",
audio=True,
seed=42,
)
assert p["prompt"] == "x"
assert p["duration"] == "8s" # veo3.1 uses "Ns" format per FAL API
assert p["aspect_ratio"] == "16:9"
assert p["resolution"] == "720p"
assert p["generate_audio"] is True
assert p["negative_prompt"] == "ugly"
assert p["seed"] == 42
def test_pixverse_range_clamps_correctly(self):
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["pixverse-v6"]
p = _build_payload(
meta,
prompt="x",
image_url="https://i.png",
duration=99, # over max → 15
aspect_ratio="16:9",
resolution="540p",
negative_prompt=None,
audio=None,
seed=None,
)
assert p["duration"] == "15"
def test_kling_4k_clamps_below_min(self):
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["kling-v3-4k"]
p = _build_payload(
meta,
prompt="x",
image_url="https://i.png",
duration=1, # below min (3) → 3
aspect_ratio="16:9",
resolution="720p",
negative_prompt=None,
audio=None,
seed=None,
)
assert p["duration"] == "3"
def test_ltx_omits_duration_aspect_resolution(self):
"""LTX 2.3 doesn't declare duration/aspect/resolution enums —
the payload should NOT include those keys (let FAL default)."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["ltx-2.3"]
p = _build_payload(
meta,
prompt="x",
image_url=None,
duration=8,
aspect_ratio="16:9",
resolution="720p",
negative_prompt="ugly",
audio=True,
seed=None,
)
assert "duration" not in p
assert "aspect_ratio" not in p
assert "resolution" not in p
# But audio + negative are advertised
assert p["generate_audio"] is True
assert p["negative_prompt"] == "ugly"
def test_happy_horse_minimal_payload(self):
"""Happy Horse has sparse docs — payload should be minimal."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["happy-horse"]
p = _build_payload(
meta,
prompt="a horse galloping",
image_url=None,
duration=8,
aspect_ratio="16:9",
resolution="720p",
negative_prompt="watermark",
audio=True,
seed=None,
)
# Only prompt — no payload bloat for fields we can't verify
assert p == {"prompt": "a horse galloping"}
+113
View File
@@ -0,0 +1,113 @@
"""Smoke tests for the xAI video gen plugin — load & register surface."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
def test_xai_provider_registers():
from plugins.video_gen.xai import XAIVideoGenProvider
provider = XAIVideoGenProvider()
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("xai") is provider
assert provider.display_name == "xAI"
assert provider.default_model() == "grok-imagine-video"
def test_xai_capabilities_text_and_image_only():
"""xAI was previously advertised with edit/extend operations. The
simplified surface only exposes text-to-video and image-to-video
confirm those are the only modalities advertised."""
from plugins.video_gen.xai import XAIVideoGenProvider
caps = XAIVideoGenProvider().capabilities()
assert caps["modalities"] == ["text", "image"]
# No 'operations' key in the simplified surface
assert "operations" not in caps
assert caps["max_reference_images"] == 7
def test_xai_unavailable_without_key(monkeypatch):
from plugins.video_gen.xai import XAIVideoGenProvider
monkeypatch.delenv("XAI_API_KEY", raising=False)
assert XAIVideoGenProvider().is_available() is False
def test_xai_generate_requires_xai_key(monkeypatch):
from plugins.video_gen.xai import XAIVideoGenProvider
monkeypatch.delenv("XAI_API_KEY", raising=False)
result = XAIVideoGenProvider().generate("a happy dog")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_xai_available_with_oauth_only(monkeypatch):
"""The plugin must honour xAI Grok OAuth credentials, not just
XAI_API_KEY. Otherwise the agent's tool-availability check filters
``video_generate`` out of the toolbelt and the agent silently falls
back to whatever skill advertises video generation (e.g. comfyui).
"""
import plugins.video_gen.xai as xai_plugin
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(
"tools.xai_http.resolve_xai_http_credentials",
lambda: {
"provider": "xai-oauth",
"api_key": "oauth-bearer-token",
"base_url": "https://api.x.ai/v1",
},
)
assert xai_plugin.XAIVideoGenProvider().is_available() is True
def test_xai_resolved_credentials_threaded_through_request(monkeypatch):
"""OAuth-resolved creds must reach the HTTP layer — bug class where
``is_available()`` says yes but the request still hits with no key.
"""
import plugins.video_gen.xai as xai_plugin
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(
"tools.xai_http.resolve_xai_http_credentials",
lambda: {
"provider": "xai-oauth",
"api_key": "oauth-bearer-token",
"base_url": "https://api.x.ai/v1",
},
)
api_key, base_url = xai_plugin._resolve_xai_credentials()
assert api_key == "oauth-bearer-token"
assert base_url == "https://api.x.ai/v1"
headers = xai_plugin._xai_headers(api_key)
assert headers["Authorization"] == "Bearer oauth-bearer-token"
def test_xai_no_operation_kwarg():
"""The ABC's generate() signature no longer accepts 'operation'.
Passing it through **kwargs should be ignored (forward-compat)."""
from plugins.video_gen.xai import XAIVideoGenProvider
# We're not actually hitting the network — just verify the call
# doesn't TypeError on the unexpected kwarg.
# Will fail with auth_required (no XAI_API_KEY), but should NOT
# fail with TypeError.
result = XAIVideoGenProvider().generate("x", operation="generate")
assert result["success"] is False
# auth_required, NOT some signature error
assert result["error_type"] in {"auth_required", "api_error"}
@@ -0,0 +1,191 @@
"""Integration tests for the xAI video gen plugin's simplified surface.
xAI exposes only text-to-video and image-to-video through the unified
``video_generate`` tool. We assert the endpoint hit and the payload shape
because routing is the part most likely to break silently.
"""
from __future__ import annotations
import asyncio
import json
from typing import Any, Dict, List, Optional
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
class _FakeResponse:
def __init__(self, status: int = 200, payload: Optional[Dict[str, Any]] = None):
self.status_code = status
self._payload = payload or {}
self.text = json.dumps(self._payload)
def raise_for_status(self):
if self.status_code >= 400:
import httpx
raise httpx.HTTPStatusError("err", request=None, response=self) # type: ignore
def json(self):
return self._payload
class _FakeAsyncClient:
def __init__(self):
self.posts: List[Dict[str, Any]] = []
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return None
async def post(self, url, headers=None, json=None, timeout=None):
self.posts.append({"url": url, "json": json})
return _FakeResponse(200, {"request_id": "req-123"})
async def get(self, url, headers=None, timeout=None):
return _FakeResponse(200, {
"status": "done",
"video": {"url": "https://xai-cdn/out.mp4", "duration": 8},
"model": "grok-imagine-video",
})
@pytest.fixture
def xai_provider(monkeypatch):
monkeypatch.setenv("XAI_API_KEY", "test-key")
import plugins.video_gen.xai as xai_plugin
captured: Dict[str, _FakeAsyncClient] = {}
def _client_factory():
captured["client"] = _FakeAsyncClient()
return captured["client"]
monkeypatch.setattr(xai_plugin.httpx, "AsyncClient", _client_factory)
async def _no_sleep(*a, **k):
return None
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
provider = xai_plugin.XAIVideoGenProvider()
return provider, captured
def _last_post(captured) -> Dict[str, Any]:
return captured["client"].posts[-1]
class TestXAIEndpoint:
"""xAI uses one endpoint — ``/videos/generations`` — for both modes."""
def test_text_to_video_hits_generations(self, xai_provider):
provider, captured = xai_provider
result = provider.generate("a dog on a skateboard")
assert result["success"] is True
assert _last_post(captured)["url"].endswith("/videos/generations")
assert result["modality"] == "text"
def test_image_to_video_hits_generations(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"animate this",
image_url="https://example.com/cat.png",
)
assert result["success"] is True
assert _last_post(captured)["url"].endswith("/videos/generations")
assert result["modality"] == "image"
class TestXAIPayload:
def test_text_payload_has_no_image_field(self, xai_provider):
provider, captured = xai_provider
provider.generate("a dog at sunset")
payload = _last_post(captured)["json"]
assert payload["prompt"] == "a dog at sunset"
assert "image" not in payload
assert "reference_images" not in payload
def test_image_payload_has_image_field(self, xai_provider):
provider, captured = xai_provider
provider.generate("animate this", image_url="https://example.com/cat.png")
payload = _last_post(captured)["json"]
assert payload["image"] == {"url": "https://example.com/cat.png"}
def test_reference_images_payload(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"keep this character",
reference_image_urls=[
"https://example.com/a.png",
"https://example.com/b.png",
],
)
payload = _last_post(captured)["json"]
assert payload["reference_images"] == [
{"url": "https://example.com/a.png"},
{"url": "https://example.com/b.png"},
]
class TestXAIValidation:
def test_missing_prompt_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate("")
assert result["success"] is False
assert result["error_type"] == "missing_prompt"
# Never hit the network
assert "client" not in captured or not captured["client"].posts
def test_image_plus_refs_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"x",
image_url="https://example.com/i.png",
reference_image_urls=["https://example.com/r.png"],
)
assert result["success"] is False
assert result["error_type"] == "conflicting_inputs"
assert "client" not in captured or not captured["client"].posts
def test_too_many_references_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"x",
reference_image_urls=[f"https://example.com/r{i}.png" for i in range(8)],
)
assert result["success"] is False
assert result["error_type"] == "too_many_references"
class TestXAIClamping:
def test_duration_clamped_to_15(self, xai_provider):
provider, captured = xai_provider
provider.generate("x", duration=30)
assert _last_post(captured)["json"]["duration"] == 15
def test_duration_clamped_when_refs_present(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"x",
duration=15,
reference_image_urls=["https://example.com/r.png"],
)
# refs present caps to 10
assert _last_post(captured)["json"]["duration"] == 10
def test_invalid_aspect_ratio_soft_clamps(self, xai_provider):
provider, captured = xai_provider
provider.generate("x", aspect_ratio="21:9")
assert _last_post(captured)["json"]["aspect_ratio"] == "16:9"
View File
@@ -0,0 +1,498 @@
"""Plugin-side tests for the web search provider migration (PR #25182).
Covers:
- All eight bundled plugins (brave-free, ddgs, searxng, exa, parallel,
tavily, firecrawl, xai) instantiate and self-report the expected
capabilities + ABC-derived defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The web_search_registry resolves an active provider in the documented
scenarios (explicit config wins ignoring availability, fallback walks
legacy preference filtered by availability, unknown name falls back).
- Plugin response shapes match the legacy bit-for-bit contract.
Per the dev skill: these tests use *real* imports from the plugin
modules no mocking of provider classes themselves so the test
catches drift in the ABC interface, the registry, and the plugin
glue layer simultaneously.
"""
from __future__ import annotations
import asyncio
import inspect
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every web-provider env var so is_available() returns False."""
for k in (
"BRAVE_SEARCH_API_KEY",
"SEARXNG_URL",
"TAVILY_API_KEY",
"TAVILY_BASE_URL",
"EXA_API_KEY",
"PARALLEL_API_KEY",
"PARALLEL_SEARCH_MODE",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_USER_TOKEN",
"XAI_API_KEY",
):
monkeypatch.delenv(k, raising=False)
def _ensure_plugins_loaded() -> None:
"""Idempotently load plugins so the registry is populated."""
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
# ---------------------------------------------------------------------------
# Per-plugin discovery + capability flags
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each test starts with a clean web-provider env."""
_clear_web_env(monkeypatch)
class TestBundledPluginsRegister:
"""All eight bundled web plugins discover and register correctly."""
def test_all_seven_plugins_present_in_registry(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import list_providers
names = sorted(p.name for p in list_providers())
assert names == [
"brave-free",
"ddgs",
"exa",
"firecrawl",
"parallel",
"searxng",
"tavily",
"xai",
]
@pytest.mark.parametrize(
"plugin_name,expected_search,expected_extract",
[
("brave-free", True, False),
("ddgs", True, False),
("searxng", True, False),
("exa", True, True),
("parallel", True, True),
("tavily", True, True),
("firecrawl", True, True),
# xai: search-only via Grok's agentic web_search tool.
("xai", True, False),
],
)
def test_capability_flags_match_spec(
self,
plugin_name: str,
expected_search: bool,
expected_extract: bool,
) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None, f"plugin {plugin_name!r} not registered"
assert provider.supports_search() is expected_search
assert provider.supports_extract() is expected_extract
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"],
)
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
assert provider.name == plugin_name
assert provider.display_name # any non-empty string
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"],
)
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
"""``get_setup_schema()`` returns a dict the picker can consume."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
schema = provider.get_setup_schema()
assert isinstance(schema, dict)
assert "name" in schema
assert "env_vars" in schema
# ---------------------------------------------------------------------------
# is_available() behavior
# ---------------------------------------------------------------------------
class TestIsAvailable:
"""Each plugin's ``is_available()`` returns False without env config."""
def test_brave_free_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("brave-free")
assert p is not None
assert p.is_available() is False # no BRAVE_SEARCH_API_KEY
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "real")
assert p.is_available() is True
def test_searxng_requires_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("searxng")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080")
assert p.is_available() is True
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("TAVILY_API_KEY", "real")
assert p.is_available() is True
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("EXA_API_KEY", "real")
assert p.is_available() is True
def test_parallel_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("PARALLEL_API_KEY", "real")
assert p.is_available() is True
def test_firecrawl_requires_either_key_or_url(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert p.is_available() is False
# Either FIRECRAWL_API_KEY or FIRECRAWL_API_URL lights it up.
monkeypatch.setenv("FIRECRAWL_API_KEY", "real")
assert p.is_available() is True
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
monkeypatch.setenv("FIRECRAWL_API_URL", "http://localhost:3002")
assert p.is_available() is True
def test_ddgs_always_available_when_package_importable(self) -> None:
"""DDGS is the always-on fallback — no API key required.
It may report unavailable if the ``ddgs`` package itself isn't
installed in the env (legitimate the plugin's post_setup hook
triggers pip install on first selection). We only assert that
is_available() doesn't raise.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("ddgs")
assert p is not None
# Truthy or falsy, just must not raise.
_ = bool(p.is_available())
def test_xai_requires_api_key_or_oauth(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""xAI needs XAI_API_KEY or OAuth tokens in auth.json."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("xai")
assert p is not None
assert p.is_available() is False # no XAI_API_KEY, no auth.json
monkeypatch.setenv("XAI_API_KEY", "real")
assert p.is_available() is True
# ---------------------------------------------------------------------------
# Registry resolution semantics (Option B — conservative smart fallback)
# ---------------------------------------------------------------------------
class TestRegistryResolution:
"""``_resolve()`` follows explicit-config + availability-filtered fallback."""
def test_explicit_configured_provider_returned_even_when_unavailable(
self,
) -> None:
"""Explicit ``web.search_backend`` wins regardless of is_available().
Without availability filtering on the explicit path, the dispatcher
would silently switch backends; with this check the dispatcher
surfaces a precise "FOO_API_KEY is not set" error instead.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
# No BRAVE_SEARCH_API_KEY (fixture cleared it).
result = _resolve("brave-free", capability="search")
assert result is not None
assert result.name == "brave-free"
# Confirm it's the unavailable one — dispatcher will surface
# a typed credential-missing error to the caller.
assert result.is_available() is False
def test_unknown_configured_name_falls_back_to_available_provider(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Typo / uninstalled plugin → walk legacy preference, pick available."""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
monkeypatch.setenv("EXA_API_KEY", "real")
result = _resolve("not-a-real-provider", capability="search")
# Either ddgs (no-key fallback) or exa (the only available
# premium provider) — both are valid. The point is the unknown
# name shouldn't return None when SOMETHING is available.
assert result is not None
assert result.is_available() is True
def test_explicit_search_only_provider_for_extract_falls_back(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Asking for extract via a search-only backend → fall back.
``brave-free`` is search-only (``supports_extract() is False``).
When the registry resolves it for an extract capability, the
explicit-config branch rejects it as capability-incompatible
and the fallback walk picks an extract-capable provider.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
monkeypatch.setenv("EXA_API_KEY", "real")
result = _resolve("brave-free", capability="extract")
# Should land on exa (only extract-capable available provider).
assert result is not None
assert result.supports_extract() is True
assert result.is_available() is True
def test_no_config_no_credentials_returns_none(
self,
) -> None:
"""No backend configured AND no available providers → typically None.
``ddgs`` is the no-credential fallback; if its ``ddgs`` Python
package is installed in the test env, ddgs will be picked.
Otherwise the resolver returns None. Either outcome is correct.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
result = _resolve(None, capability="search")
if result is not None:
# The only no-credential provider is ddgs; anything else
# means an env var leaked in.
assert result.is_available() is True
# ---------------------------------------------------------------------------
# Sync-vs-async extract detection
# ---------------------------------------------------------------------------
class TestAsyncExtractDispatch:
"""The dispatcher detects async vs sync extract methods correctly."""
def test_parallel_extract_is_async(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is True
def test_firecrawl_extract_is_async(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is True
def test_exa_extract_is_sync(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is False
def test_tavily_extract_is_sync(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is False
# ---------------------------------------------------------------------------
# Error response shape (preserved bit-for-bit from legacy)
# ---------------------------------------------------------------------------
class TestErrorResponseShapes:
"""When credentials are missing, plugins return typed errors, not raises."""
def test_brave_free_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("brave-free")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_searxng_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("searxng")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_exa_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_tavily_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_parallel_extract_returns_per_url_errors_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
result = asyncio.run(p.extract(["https://example.com"]))
assert isinstance(result, list)
assert len(result) == 1
assert "error" in result[0]
assert result[0]["url"] == "https://example.com"
def test_firecrawl_extract_returns_per_url_errors_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
# firecrawl extract returns [] when the website-policy gate rejects
# the URL, or a per-URL error dict when the gate passes but the
# firecrawl client fails. Use a URL the policy allows to make sure
# we hit the credential-missing path.
result = asyncio.run(p.extract(["https://example.com"]))
assert isinstance(result, list)
if result: # if anything came back, it should be an error entry
assert "error" in result[0]
def test_firecrawl_config_error_points_paid_users_to_nous_subscription(self, monkeypatch):
from plugins.web.firecrawl import provider as firecrawl_provider
monkeypatch.setattr(
"tools.web_tools.managed_nous_tools_enabled",
lambda: True,
raising=False,
)
with pytest.raises(ValueError) as exc_info:
firecrawl_provider._raise_web_backend_configuration_error()
message = str(exc_info.value)
assert "With your Nous subscription you can also use the Tool Gateway" in message
assert "select Nous Subscription as the web provider" in message
assert "managed Firecrawl web tools is unavailable" not in message
def test_firecrawl_config_error_uses_entitlement_message_when_not_paid(self, monkeypatch):
from plugins.web.firecrawl import provider as firecrawl_provider
monkeypatch.setattr(
"tools.web_tools.managed_nous_tools_enabled",
lambda: False,
raising=False,
)
monkeypatch.setattr(
"tools.web_tools.nous_tool_gateway_unavailable_message",
lambda capability: f"{capability} denied by test entitlement.",
raising=False,
)
with pytest.raises(ValueError) as exc_info:
firecrawl_provider._raise_web_backend_configuration_error()
assert "managed Firecrawl web tools denied by test entitlement" in str(exc_info.value)
def test_xai_search_returns_error_dict_when_unconfigured(self) -> None:
"""xAI returns a typed error dict (no XAI_API_KEY)."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("xai")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result