Initial import of NousResearch/hermes-agent
Deploy Site / deploy-vercel (push) Has been cancelled
Deploy Site / deploy-docs (push) Has been cancelled
Docker / shell lint / Lint Dockerfile (hadolint) (push) Has been cancelled
Docker / shell lint / Lint docker/ shell scripts (shellcheck) (push) Has been cancelled
Docker Build and Publish / build-amd64 (push) Has been cancelled
Docker Build and Publish / build-arm64 (push) Has been cancelled
Lint (ruff + ty) / ruff + ty diff (push) Has been cancelled
Lint (ruff + ty) / ruff enforcement (blocking) (push) Has been cancelled
Lint (ruff + ty) / Windows footguns (blocking) (push) Has been cancelled
Nix Lockfile Fix / auto-fix-main (push) Has been cancelled
Nix Lockfile Fix / fix (push) Has been cancelled
Nix / nix (macos-latest) (push) Has been cancelled
Nix / nix (ubuntu-latest) (push) Has been cancelled
OSV-Scanner / Scan lockfiles (push) Has been cancelled
Build Skills Index / build-index (push) Has been cancelled
Tests / test (1) (push) Has been cancelled
Tests / test (2) (push) Has been cancelled
Tests / test (3) (push) Has been cancelled
Tests / test (4) (push) Has been cancelled
Tests / test (5) (push) Has been cancelled
Tests / test (6) (push) Has been cancelled
Tests / e2e (push) Has been cancelled
uv.lock check / uv lock --check (push) Has been cancelled
Docker Build and Publish / merge (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
Tests / save-durations (push) Has been cancelled
Deploy Site / deploy-vercel (push) Has been cancelled
Deploy Site / deploy-docs (push) Has been cancelled
Docker / shell lint / Lint Dockerfile (hadolint) (push) Has been cancelled
Docker / shell lint / Lint docker/ shell scripts (shellcheck) (push) Has been cancelled
Docker Build and Publish / build-amd64 (push) Has been cancelled
Docker Build and Publish / build-arm64 (push) Has been cancelled
Lint (ruff + ty) / ruff + ty diff (push) Has been cancelled
Lint (ruff + ty) / ruff enforcement (blocking) (push) Has been cancelled
Lint (ruff + ty) / Windows footguns (blocking) (push) Has been cancelled
Nix Lockfile Fix / auto-fix-main (push) Has been cancelled
Nix Lockfile Fix / fix (push) Has been cancelled
Nix / nix (macos-latest) (push) Has been cancelled
Nix / nix (ubuntu-latest) (push) Has been cancelled
OSV-Scanner / Scan lockfiles (push) Has been cancelled
Build Skills Index / build-index (push) Has been cancelled
Tests / test (1) (push) Has been cancelled
Tests / test (2) (push) Has been cancelled
Tests / test (3) (push) Has been cancelled
Tests / test (4) (push) Has been cancelled
Tests / test (5) (push) Has been cancelled
Tests / test (6) (push) Has been cancelled
Tests / e2e (push) Has been cancelled
uv.lock check / uv lock --check (push) Has been cancelled
Docker Build and Publish / merge (push) Has been cancelled
Build Skills Index / trigger-deploy (push) Has been cancelled
Tests / save-durations (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
"""Behavior-parity check for the browser-provider plugin migration (#25214).
|
||||
|
||||
Spawns one subprocess per (version, scenario) cell — pinned to either
|
||||
origin/main (legacy in-tree providers + class-instantiation lookup) or
|
||||
this PR's worktree (plugin-based registry) via `sys.path[0]`. Each
|
||||
subprocess clears all browser-related env vars + writes a config.yaml,
|
||||
loads `tools.browser_tool._get_cloud_provider()`, and emits a reduced
|
||||
"shape tuple" {is_local, provider_name, is_available} as JSON.
|
||||
|
||||
The parent process diffs the shapes per scenario. A diff means the
|
||||
migration introduced an observable behaviour change vs origin/main —
|
||||
which would be a real regression for users on the existing config keys.
|
||||
|
||||
Run from the PR worktree:
|
||||
|
||||
cd ~/.hermes/hermes-agent/.worktrees/browser-providers-plugin
|
||||
python tests/plugins/browser/check_parity_vs_main.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
# Pin one path to current main, one to the PR worktree.
|
||||
# ``REPO_ROOT`` is ``.../.worktrees/browser-providers-plugin``; the main
|
||||
# checkout lives two levels up at ``~/.hermes/hermes-agent``.
|
||||
MAIN_DIR = REPO_ROOT.parent.parent # ~/.hermes/hermes-agent
|
||||
PR_DIR = REPO_ROOT # the worktree we're in
|
||||
assert (MAIN_DIR / "tools" / "browser_tool.py").exists(), (
|
||||
f"MAIN_DIR={MAIN_DIR} doesn't look like a hermes-agent checkout"
|
||||
)
|
||||
assert (PR_DIR / "tools" / "browser_tool.py").exists(), (
|
||||
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
|
||||
)
|
||||
|
||||
|
||||
# Reduced shape comparison — exact instance addresses obviously differ
|
||||
# between subprocesses, so we compare the parts that matter for users.
|
||||
SUBPROCESS_SCRIPT = r"""
|
||||
import json, os, sys, tempfile
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
|
||||
# Isolated HERMES_HOME for the config write.
|
||||
home = tempfile.mkdtemp()
|
||||
os.environ["HERMES_HOME"] = home
|
||||
|
||||
# Clear every browser-related env var so is_available() is deterministic.
|
||||
for k in (
|
||||
"BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", "BROWSERBASE_BASE_URL",
|
||||
"BROWSER_USE_API_KEY", "BROWSER_USE_GATEWAY_URL",
|
||||
"FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_BROWSER_TTL",
|
||||
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
|
||||
):
|
||||
os.environ.pop(k, None)
|
||||
|
||||
# Apply per-scenario env (passed as JSON via argv[2]).
|
||||
scenario_env = json.loads(sys.argv[2])
|
||||
os.environ.update(scenario_env)
|
||||
|
||||
# Apply per-scenario config (passed as YAML body via argv[3]).
|
||||
config_yaml = sys.argv[3]
|
||||
config_path = os.path.join(home, "config.yaml")
|
||||
with open(config_path, "w") as f:
|
||||
f.write(config_yaml)
|
||||
|
||||
# Fresh import — must not have any browser modules cached.
|
||||
for name in list(sys.modules):
|
||||
if name.startswith("tools.") or name.startswith("agent.") or name.startswith("plugins."):
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
from tools.browser_tool import _get_cloud_provider, _is_local_mode
|
||||
|
||||
provider = _get_cloud_provider()
|
||||
|
||||
# Pull the human-readable backend name via the API that exists on BOTH
|
||||
# legacy (origin/main: CloudBrowserProvider.provider_name()) and the new
|
||||
# ABC (BrowserProvider exposes provider_name() as a backward-compat alias
|
||||
# returning display_name). Both shapes resolve to the same string —
|
||||
# 'Browserbase' / 'Browser Use' / 'Firecrawl' — so we can compare safely.
|
||||
provider_name = None
|
||||
is_available = None
|
||||
if provider is not None:
|
||||
pn = getattr(provider, "provider_name", None)
|
||||
if callable(pn):
|
||||
provider_name = pn()
|
||||
elif isinstance(pn, str):
|
||||
provider_name = pn
|
||||
is_conf = getattr(provider, "is_configured", None)
|
||||
if callable(is_conf):
|
||||
is_available = bool(is_conf())
|
||||
|
||||
shape = {
|
||||
"is_local": _is_local_mode(),
|
||||
"provider_name": provider_name,
|
||||
"is_available": is_available,
|
||||
}
|
||||
print(json.dumps(shape))
|
||||
"""
|
||||
|
||||
|
||||
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
|
||||
# (label, config.yaml body, extra env vars)
|
||||
("no-config-no-env", "", {}),
|
||||
("explicit-local-no-env", "browser:\n cloud_provider: local\n", {}),
|
||||
(
|
||||
"explicit-browserbase-no-creds",
|
||||
"browser:\n cloud_provider: browserbase\n",
|
||||
{},
|
||||
),
|
||||
(
|
||||
"explicit-browserbase-with-creds",
|
||||
"browser:\n cloud_provider: browserbase\n",
|
||||
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
|
||||
),
|
||||
(
|
||||
"explicit-browser-use-no-creds",
|
||||
"browser:\n cloud_provider: browser-use\n",
|
||||
{},
|
||||
),
|
||||
(
|
||||
"explicit-browser-use-with-creds",
|
||||
"browser:\n cloud_provider: browser-use\n",
|
||||
{"BROWSER_USE_API_KEY": "k"},
|
||||
),
|
||||
(
|
||||
"explicit-firecrawl-no-creds",
|
||||
"browser:\n cloud_provider: firecrawl\n",
|
||||
{},
|
||||
),
|
||||
(
|
||||
"explicit-firecrawl-with-creds",
|
||||
"browser:\n cloud_provider: firecrawl\n",
|
||||
{"FIRECRAWL_API_KEY": "k"},
|
||||
),
|
||||
(
|
||||
"no-config-bu-creds",
|
||||
"",
|
||||
{"BROWSER_USE_API_KEY": "k"},
|
||||
),
|
||||
(
|
||||
"no-config-bb-creds",
|
||||
"",
|
||||
{"BROWSERBASE_API_KEY": "x", "BROWSERBASE_PROJECT_ID": "y"},
|
||||
),
|
||||
(
|
||||
"no-config-both-creds",
|
||||
"",
|
||||
{
|
||||
"BROWSER_USE_API_KEY": "k",
|
||||
"BROWSERBASE_API_KEY": "x",
|
||||
"BROWSERBASE_PROJECT_ID": "y",
|
||||
},
|
||||
),
|
||||
(
|
||||
"no-config-firecrawl-only",
|
||||
"",
|
||||
{"FIRECRAWL_API_KEY": "k"},
|
||||
),
|
||||
(
|
||||
"no-config-firecrawl-and-bb",
|
||||
"",
|
||||
{
|
||||
"FIRECRAWL_API_KEY": "k",
|
||||
"BROWSERBASE_API_KEY": "x",
|
||||
"BROWSERBASE_PROJECT_ID": "y",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
|
||||
"""Run one (version, scenario) cell. Returns the shape dict."""
|
||||
venv_python = repo_path / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
# Worktrees share the main repo's venv.
|
||||
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = Path("python3")
|
||||
|
||||
out = subprocess.run(
|
||||
[
|
||||
str(venv_python),
|
||||
"-c",
|
||||
SUBPROCESS_SCRIPT,
|
||||
str(repo_path),
|
||||
json.dumps(env),
|
||||
config_yaml,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return {
|
||||
"error": "subprocess failed",
|
||||
"stdout": out.stdout,
|
||||
"stderr": out.stderr[-500:],
|
||||
}
|
||||
try:
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
except Exception as exc:
|
||||
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
|
||||
|
||||
|
||||
def _reduce_for_comparison(shape: dict) -> dict:
|
||||
"""Reduce a shape dict to the parts that matter for user-visible parity.
|
||||
|
||||
We compare ``(is_local, provider_name, is_available)`` — the trio that
|
||||
decides what the dispatcher does with each tool call. ``provider_name``
|
||||
is the legacy ``provider_name()`` return value ('Browserbase' / 'Browser
|
||||
Use' / 'Firecrawl'), which is identical between legacy and plugin
|
||||
classes (the plugin's ``display_name`` matches the legacy
|
||||
``provider_name()`` return).
|
||||
"""
|
||||
return {
|
||||
"is_local": shape.get("is_local"),
|
||||
"provider_name": shape.get("provider_name"),
|
||||
"is_available": shape.get("is_available"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print(f"main: {MAIN_DIR}")
|
||||
print(f"pr: {PR_DIR}")
|
||||
print()
|
||||
|
||||
failures: list[str] = []
|
||||
errors: list[str] = []
|
||||
for label, config_yaml, env in SCENARIOS:
|
||||
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
|
||||
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
|
||||
|
||||
if "error" in main_shape or "error" in pr_shape:
|
||||
print(f" [ERR ] {label}: subprocess failed")
|
||||
print(f" main: {main_shape}")
|
||||
print(f" pr: {pr_shape}")
|
||||
errors.append(label)
|
||||
continue
|
||||
|
||||
main_reduced = _reduce_for_comparison(main_shape)
|
||||
pr_reduced = _reduce_for_comparison(pr_shape)
|
||||
|
||||
if main_reduced == pr_reduced:
|
||||
print(f" [OK] {label}: {main_reduced}")
|
||||
else:
|
||||
print(f" [FAIL] {label}")
|
||||
print(f" main: {main_reduced}")
|
||||
print(f" pr: {pr_reduced}")
|
||||
failures.append(label)
|
||||
|
||||
print()
|
||||
if errors:
|
||||
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
if failures:
|
||||
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
|
||||
for f in failures:
|
||||
print(f" - {f}")
|
||||
if failures or errors:
|
||||
return 1
|
||||
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Plugin-side tests for the browser provider migration (PR #25214).
|
||||
|
||||
Covers:
|
||||
|
||||
- All three bundled plugins (browserbase, browser-use, firecrawl)
|
||||
instantiate and self-report the expected ABC defaults.
|
||||
- Each plugin's ``is_available()`` correctly reflects env-var presence.
|
||||
- The browser_registry resolves an active provider in the documented
|
||||
scenarios:
|
||||
* explicit config wins ignoring availability (so dispatcher surfaces
|
||||
a typed credentials error)
|
||||
* legacy preference walk: browser-use → browserbase (filtered by
|
||||
availability)
|
||||
* firecrawl is NOT in the legacy walk — explicit-only
|
||||
* unknown name falls through to auto-detect
|
||||
* ``local`` short-circuits to None
|
||||
|
||||
These tests use *real* imports from the plugin modules — no mocking of
|
||||
provider classes themselves — so the test catches drift in the ABC
|
||||
interface, the registry, and the plugin glue layer simultaneously.
|
||||
Mirrors ``tests/plugins/web/test_web_search_provider_plugins.py`` from
|
||||
PR #25182.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _clear_browser_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Strip every browser-provider env var so is_available() returns False."""
|
||||
for k in (
|
||||
"BROWSERBASE_API_KEY",
|
||||
"BROWSERBASE_PROJECT_ID",
|
||||
"BROWSERBASE_BASE_URL",
|
||||
"BROWSER_USE_API_KEY",
|
||||
"BROWSER_USE_GATEWAY_URL",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
"FIRECRAWL_BROWSER_TTL",
|
||||
"TOOL_GATEWAY_DOMAIN",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
|
||||
def _ensure_plugins_loaded() -> None:
|
||||
"""Idempotently load plugins so the registry is populated."""
|
||||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||||
|
||||
_ensure_plugins_discovered()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-test isolation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Each test starts with a clean browser-provider env."""
|
||||
_clear_browser_env(monkeypatch)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bundled plugins register
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBundledPluginsRegister:
|
||||
"""All three bundled browser plugins discover and register correctly."""
|
||||
|
||||
def test_all_three_plugins_present_in_registry(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import list_providers
|
||||
|
||||
names = sorted(p.name for p in list_providers())
|
||||
assert names == ["browser-use", "browserbase", "firecrawl"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name,expected_display",
|
||||
[
|
||||
("browserbase", "Browserbase"),
|
||||
("browser-use", "Browser Use"),
|
||||
("firecrawl", "Firecrawl"),
|
||||
],
|
||||
)
|
||||
def test_each_plugin_has_name_and_display_name(
|
||||
self, plugin_name: str, expected_display: str
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None, f"plugin {plugin_name!r} not registered"
|
||||
assert provider.name == plugin_name
|
||||
assert provider.display_name == expected_display
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["browserbase", "browser-use", "firecrawl"],
|
||||
)
|
||||
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
|
||||
"""``get_setup_schema()`` returns a dict the picker can consume."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
schema = provider.get_setup_schema()
|
||||
assert isinstance(schema, dict)
|
||||
assert "name" in schema
|
||||
assert "env_vars" in schema
|
||||
# Every cloud-browser plugin needs the agent-browser post-setup hook
|
||||
# so the picker auto-installs the CLI on selection.
|
||||
assert schema.get("post_setup") == "agent_browser"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["browserbase", "browser-use", "firecrawl"],
|
||||
)
|
||||
def test_each_plugin_implements_full_lifecycle(self, plugin_name: str) -> None:
|
||||
"""The ABC's three lifecycle methods are all overridden."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_provider import BrowserProvider
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
# Each method must be a real override, not the ABC's NotImplementedError
|
||||
# default — we check by comparing the function reference.
|
||||
assert type(provider).create_session is not BrowserProvider.create_session
|
||||
assert type(provider).close_session is not BrowserProvider.close_session
|
||||
assert (
|
||||
type(provider).emergency_cleanup is not BrowserProvider.emergency_cleanup
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_available() behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsAvailable:
|
||||
"""Each plugin's ``is_available()`` reflects env-var presence accurately."""
|
||||
|
||||
def test_browserbase_requires_both_api_key_and_project_id(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
p = get_provider("browserbase")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
|
||||
# API key alone is insufficient.
|
||||
monkeypatch.setenv("BROWSERBASE_API_KEY", "key")
|
||||
assert p.is_available() is False
|
||||
|
||||
# Both env vars set → available.
|
||||
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_browserbase_project_id_alone_insufficient(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
p = get_provider("browserbase")
|
||||
assert p is not None
|
||||
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj")
|
||||
assert p.is_available() is False
|
||||
|
||||
def test_browser_use_satisfied_by_api_key(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
p = get_provider("browser-use")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "key")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_firecrawl_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("FIRECRAWL_API_KEY", "key")
|
||||
assert p.is_available() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry resolution semantics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryResolution:
|
||||
"""``_resolve()`` implements the documented three-rule precedence."""
|
||||
|
||||
def test_resolve_none_with_no_creds_returns_none(self) -> None:
|
||||
"""No config, no env → local mode (None)."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
assert _resolve(None) is None
|
||||
|
||||
def test_explicit_local_returns_none(self) -> None:
|
||||
"""``cloud_provider: local`` is a positive choice; short-circuits to None."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
assert _resolve("local") is None
|
||||
|
||||
def test_explicit_browserbase_returns_provider_even_when_unavailable(self) -> None:
|
||||
"""Rule 1: explicit-config wins even when credentials are missing.
|
||||
|
||||
This is critical — the dispatcher needs to surface a typed
|
||||
credentials error rather than silently switching backends.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
provider = _resolve("browserbase")
|
||||
assert provider is not None
|
||||
assert provider.name == "browserbase"
|
||||
assert provider.is_available() is False # confirms "ignoring availability"
|
||||
|
||||
def test_explicit_firecrawl_returns_provider_even_when_unavailable(self) -> None:
|
||||
"""Firecrawl behaves the same as browserbase under explicit config."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
provider = _resolve("firecrawl")
|
||||
assert provider is not None
|
||||
assert provider.name == "firecrawl"
|
||||
|
||||
def test_explicit_unknown_falls_back_to_auto_detect(self) -> None:
|
||||
"""Rule 1 miss: unknown name → fall through to legacy walk."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
# With no credentials anywhere, auto-detect should also fail.
|
||||
assert _resolve("not-a-real-provider") is None
|
||||
|
||||
def test_legacy_walk_prefers_browser_use_over_browserbase(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Rule 3: walk order is browser-use → browserbase."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
# Both available — browser-use should win.
|
||||
monkeypatch.setenv("BROWSER_USE_API_KEY", "k1")
|
||||
monkeypatch.setenv("BROWSERBASE_API_KEY", "k2")
|
||||
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
|
||||
|
||||
provider = _resolve(None)
|
||||
assert provider is not None
|
||||
assert provider.name == "browser-use"
|
||||
|
||||
def test_legacy_walk_falls_through_to_browserbase(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Rule 3: browser-use unavailable → browserbase picked."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
monkeypatch.setenv("BROWSERBASE_API_KEY", "k")
|
||||
monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "p")
|
||||
|
||||
provider = _resolve(None)
|
||||
assert provider is not None
|
||||
assert provider.name == "browserbase"
|
||||
|
||||
def test_firecrawl_not_in_legacy_walk_even_when_only_one_available(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Regression: firecrawl is NEVER auto-selected even when single-eligible.
|
||||
|
||||
Pre-PR-#25214, the dispatcher only auto-detected between Browser Use
|
||||
and Browserbase; firecrawl was reachable solely via explicit
|
||||
config. We preserve that gate because FIRECRAWL_API_KEY is shared
|
||||
with the *web* firecrawl plugin — auto-routing a web-extract user
|
||||
to a paid cloud browser would be a real behaviour regression.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import _resolve
|
||||
|
||||
monkeypatch.setenv("FIRECRAWL_API_KEY", "k")
|
||||
|
||||
# Only firecrawl is_available() — but it's not in the legacy walk.
|
||||
assert _resolve(None) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legacy ABC backward-compat aliases (is_configured / provider_name)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLegacyAbcAliases:
|
||||
"""is_configured() and provider_name() delegate to the new API."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["browserbase", "browser-use", "firecrawl"],
|
||||
)
|
||||
def test_is_configured_delegates_to_is_available(self, plugin_name: str) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
p = get_provider(plugin_name)
|
||||
assert p is not None
|
||||
assert p.is_configured() is p.is_available()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name,expected_label",
|
||||
[
|
||||
("browserbase", "Browserbase"),
|
||||
("browser-use", "Browser Use"),
|
||||
("firecrawl", "Firecrawl"),
|
||||
],
|
||||
)
|
||||
def test_provider_name_returns_display_name(
|
||||
self, plugin_name: str, expected_label: str
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.browser_registry import get_provider
|
||||
|
||||
p = get_provider(plugin_name)
|
||||
assert p is not None
|
||||
assert p.provider_name() == expected_label
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Picker integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPickerIntegration:
|
||||
"""`_plugin_browser_providers()` exposes all three plugins as picker rows."""
|
||||
|
||||
def test_picker_rows_match_registered_plugins(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from hermes_cli.tools_config import _plugin_browser_providers
|
||||
|
||||
rows = _plugin_browser_providers()
|
||||
names = sorted(r.get("browser_provider") for r in rows)
|
||||
assert names == ["browser-use", "browserbase", "firecrawl"]
|
||||
|
||||
def test_picker_rows_carry_post_setup_hook(self) -> None:
|
||||
"""Every browser plugin row has post_setup='agent_browser' so
|
||||
selecting it triggers the agent-browser CLI install."""
|
||||
_ensure_plugins_loaded()
|
||||
from hermes_cli.tools_config import _plugin_browser_providers
|
||||
|
||||
for row in _plugin_browser_providers():
|
||||
assert row.get("post_setup") == "agent_browser", (
|
||||
f"plugin row {row['browser_provider']!r} missing post_setup hook"
|
||||
)
|
||||
|
||||
def test_picker_rows_carry_browser_plugin_name_marker(self) -> None:
|
||||
"""`browser_plugin_name` matches `browser_provider` so downstream
|
||||
code can route through the registry when it wants to."""
|
||||
_ensure_plugins_loaded()
|
||||
from hermes_cli.tools_config import _plugin_browser_providers
|
||||
|
||||
for row in _plugin_browser_providers():
|
||||
assert row.get("browser_plugin_name") == row.get("browser_provider")
|
||||
@@ -0,0 +1,755 @@
|
||||
"""Tests for the bundled Nous dashboard-auth plugin.
|
||||
|
||||
Covers four shapes from Phase 4 of ``.hermes/plans/2026-05-21-dashboard-oauth-auth.md``:
|
||||
|
||||
1. Plugin entry-point registration gating (env var checks).
|
||||
2. ``start_login`` shape (PKCE/state, authorize URL parameters).
|
||||
3. ``complete_login`` httpx-mocked happy path + error mapping.
|
||||
4. ``verify_session`` JWT verification — RSA keypair, audience/issuer pinning,
|
||||
``agent_instance_id`` cross-check, ``oauth_contract_version`` tolerance.
|
||||
|
||||
Also exercises ``revoke_session`` (no-op) and ``refresh_session``
|
||||
(unconditional ``RefreshExpiredError``).
|
||||
|
||||
All HTTP is mocked: nothing in this file talks to a real Portal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
import urllib.parse
|
||||
from typing import Any, Dict
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import jwt
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
|
||||
import plugins.dashboard_auth.nous as nous_plugin
|
||||
from hermes_cli.dashboard_auth import (
|
||||
InvalidCodeError,
|
||||
LoginStart,
|
||||
ProviderError,
|
||||
RefreshExpiredError,
|
||||
Session,
|
||||
assert_protocol_compliance,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RSA keypair fixture (module-scope — keygen is slow)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rsa_keypair() -> Dict[str, Any]:
|
||||
"""Generate an RS256 keypair + matching JWK for verify_session tests."""
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
private_pem = key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.PKCS8,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
).decode()
|
||||
public_numbers = key.public_key().public_numbers()
|
||||
|
||||
def _b64url_uint(n: int) -> str:
|
||||
length = (n.bit_length() + 7) // 8
|
||||
return (
|
||||
base64.urlsafe_b64encode(n.to_bytes(length, "big")).rstrip(b"=").decode()
|
||||
)
|
||||
|
||||
jwk = {
|
||||
"kty": "RSA",
|
||||
"use": "sig",
|
||||
"alg": "RS256",
|
||||
"kid": "test-key-1",
|
||||
"n": _b64url_uint(public_numbers.n),
|
||||
"e": _b64url_uint(public_numbers.e),
|
||||
}
|
||||
return {"private_pem": private_pem, "jwk": jwk, "kid": jwk["kid"]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token-mint helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mint_token(
|
||||
rsa_keypair: Dict[str, Any],
|
||||
*,
|
||||
iss: str = "https://portal.example.com",
|
||||
aud: str = "agent:inst123",
|
||||
sub: str = "usr_abc",
|
||||
agent_instance_id: str | None = "inst123",
|
||||
oauth_contract_version: Any = 1,
|
||||
org_id: str | None = "org_xyz",
|
||||
scope: str = "agent_dashboard:access",
|
||||
ttl_seconds: int = 900,
|
||||
extra_claims: Dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
now = int(time.time())
|
||||
claims = {
|
||||
"iss": iss,
|
||||
"aud": aud,
|
||||
"sub": sub,
|
||||
"iat": now,
|
||||
"exp": now + ttl_seconds,
|
||||
"scope": scope,
|
||||
}
|
||||
if agent_instance_id is not None:
|
||||
claims["agent_instance_id"] = agent_instance_id
|
||||
if oauth_contract_version is not None:
|
||||
claims["oauth_contract_version"] = oauth_contract_version
|
||||
if org_id is not None:
|
||||
claims["org_id"] = org_id
|
||||
if extra_claims:
|
||||
claims.update(extra_claims)
|
||||
return jwt.encode(
|
||||
claims,
|
||||
rsa_keypair["private_pem"],
|
||||
algorithm="RS256",
|
||||
headers={"kid": rsa_keypair["kid"]},
|
||||
)
|
||||
|
||||
|
||||
def _patched_jwks(provider: nous_plugin.NousDashboardAuthProvider, rsa_keypair):
|
||||
"""Patch the provider's JWKS client to return our fixture key."""
|
||||
fake_key = MagicMock()
|
||||
fake_key.key = serialization.load_pem_private_key(
|
||||
rsa_keypair["private_pem"].encode(), password=None
|
||||
).public_key()
|
||||
fake_client = MagicMock()
|
||||
fake_client.get_signing_key_from_jwt.return_value = fake_key
|
||||
provider._jwks_client = fake_client
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConstruction:
|
||||
def test_protocol_compliance(self):
|
||||
assert_protocol_compliance(nous_plugin.NousDashboardAuthProvider)
|
||||
|
||||
def test_name_and_display(self):
|
||||
p = nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:inst1", portal_url="https://portal.example.com"
|
||||
)
|
||||
assert p.name == "nous"
|
||||
assert p.display_name == "Nous Research"
|
||||
|
||||
def test_extracts_agent_instance_id(self):
|
||||
p = nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:abc-123", portal_url="https://portal.example.com"
|
||||
)
|
||||
assert p._agent_instance_id == "abc-123"
|
||||
|
||||
def test_strips_trailing_slash_from_portal_url(self):
|
||||
p = nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:x", portal_url="https://portal.example.com/"
|
||||
)
|
||||
assert p._portal_url == "https://portal.example.com"
|
||||
|
||||
def test_rejects_malformed_client_id(self):
|
||||
with pytest.raises(ValueError, match="agent:"):
|
||||
nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="hermes-dashboard", portal_url="https://x"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point: env-gated registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPluginRegister:
|
||||
def test_skips_when_client_id_missing(self, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_not_called()
|
||||
# Skip reason is surfaced for the gate's fail-closed message.
|
||||
assert "HERMES_DASHBOARD_OAUTH_CLIENT_ID" in nous_plugin.LAST_SKIP_REASON
|
||||
|
||||
def test_registers_with_default_portal_url_when_only_client_id_set(
|
||||
self, monkeypatch
|
||||
):
|
||||
"""Phase 7 follow-up: HERMES_DASHBOARD_PORTAL_URL is optional —
|
||||
defaults to the production Nous Portal. The user shouldn't have
|
||||
to set it for the common production deployment path."""
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_called_once()
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert isinstance(registered, nous_plugin.NousDashboardAuthProvider)
|
||||
assert registered._portal_url == "https://portal.nousresearch.com"
|
||||
# Skip reason cleared on successful registration.
|
||||
assert nous_plugin.LAST_SKIP_REASON == ""
|
||||
|
||||
def test_skips_when_client_id_malformed(self, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "hermes-dashboard")
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "https://p.example")
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_not_called()
|
||||
# Skip reason names the offending value + contract shape.
|
||||
assert "agent:" in nous_plugin.LAST_SKIP_REASON
|
||||
assert "hermes-dashboard" in nous_plugin.LAST_SKIP_REASON
|
||||
|
||||
def test_registers_with_explicit_portal_url(self, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "https://p.example")
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_called_once()
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._client_id == "agent:inst1"
|
||||
assert registered._portal_url == "https://p.example"
|
||||
|
||||
def test_strips_whitespace_from_env_vars(self, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", " agent:x ")
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", " https://p.example ")
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_called_once()
|
||||
|
||||
def test_empty_portal_url_env_uses_default(self, monkeypatch):
|
||||
"""Explicit empty string still falls back to the production
|
||||
default — same handling as 'unset' so an empty Fly secret can't
|
||||
accidentally point the dashboard at nowhere."""
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:inst1")
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_PORTAL_URL", "")
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._portal_url == "https://portal.nousresearch.com"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point: config.yaml + env-override precedence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfigYamlSource:
|
||||
"""``dashboard.oauth.{client_id,portal_url}`` in ``config.yaml`` is the
|
||||
canonical surface for these settings. ``HERMES_DASHBOARD_OAUTH_CLIENT_ID``
|
||||
and ``HERMES_DASHBOARD_PORTAL_URL`` are operator overrides that win when
|
||||
set — this is the contract Fly.io's platform-secret injection relies on,
|
||||
and the contract that lets local devs experiment without setting env
|
||||
vars.
|
||||
|
||||
Each test pins exactly one tier of the precedence chain so a regression
|
||||
that flips the order is caught:
|
||||
|
||||
env (when truthy) > config.yaml (when truthy) > plugin default
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def patch_config(self, monkeypatch):
|
||||
"""Yield a callable that replaces ``hermes_cli.config.load_config``
|
||||
with a stub returning the given dict. Tests pass the intended
|
||||
``dashboard.oauth`` block; the stub returns the wrapping structure."""
|
||||
|
||||
def _set(oauth_block: Dict[str, Any] | None) -> None:
|
||||
cfg = {}
|
||||
if oauth_block is not None:
|
||||
cfg = {"dashboard": {"oauth": oauth_block}}
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.load_config", lambda: cfg
|
||||
)
|
||||
|
||||
return _set
|
||||
|
||||
def test_config_yaml_only_client_id_registers(self, patch_config, monkeypatch):
|
||||
"""No env var, only config.yaml — plugin reads from config and
|
||||
registers successfully. This is the path Teknium's review pushed
|
||||
for (".env is for secrets only")."""
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
|
||||
patch_config({"client_id": "agent:from-config"})
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_called_once()
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._client_id == "agent:from-config"
|
||||
# Defaults to production portal URL when neither config nor env
|
||||
# specifies one.
|
||||
assert registered._portal_url == "https://portal.nousresearch.com"
|
||||
|
||||
def test_config_yaml_client_id_and_portal_url(self, patch_config, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_PORTAL_URL", raising=False)
|
||||
patch_config({
|
||||
"client_id": "agent:from-config",
|
||||
"portal_url": "https://staging.portal.example",
|
||||
})
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._client_id == "agent:from-config"
|
||||
assert registered._portal_url == "https://staging.portal.example"
|
||||
|
||||
def test_env_overrides_config_client_id(self, patch_config, monkeypatch):
|
||||
"""Env wins. Critical for Fly.io: the Portal injects
|
||||
HERMES_DASHBOARD_OAUTH_CLIENT_ID at deploy time and we MUST
|
||||
honour it even if a stale config.yaml ships in the image."""
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:from-env")
|
||||
patch_config({"client_id": "agent:from-config"})
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._client_id == "agent:from-env", (
|
||||
"env var must override config.yaml — Fly secret injection "
|
||||
"depends on this precedence"
|
||||
)
|
||||
|
||||
def test_env_overrides_config_portal_url(self, patch_config, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "agent:x")
|
||||
monkeypatch.setenv(
|
||||
"HERMES_DASHBOARD_PORTAL_URL", "https://env.portal.example",
|
||||
)
|
||||
patch_config({
|
||||
"client_id": "agent:x",
|
||||
"portal_url": "https://config.portal.example",
|
||||
})
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._portal_url == "https://env.portal.example"
|
||||
|
||||
def test_empty_env_string_does_not_shadow_config(
|
||||
self, patch_config, monkeypatch
|
||||
):
|
||||
"""``HERMES_DASHBOARD_OAUTH_CLIENT_ID=`` (set but empty) is
|
||||
common in CI/Fly when a secret is provisioned-but-not-populated.
|
||||
It MUST NOT shadow a valid config.yaml value with an empty
|
||||
string — operators would lose the gate."""
|
||||
monkeypatch.setenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", "")
|
||||
patch_config({"client_id": "agent:from-config"})
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_called_once()
|
||||
registered = ctx.register_dashboard_auth_provider.call_args.args[0]
|
||||
assert registered._client_id == "agent:from-config"
|
||||
|
||||
def test_neither_source_skips_with_helpful_reason(
|
||||
self, patch_config, monkeypatch
|
||||
):
|
||||
"""Neither env nor config.yaml set — skip with a reason that
|
||||
mentions BOTH surfaces so operators don't guess wrong about
|
||||
which one to populate."""
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
|
||||
patch_config(None)
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_not_called()
|
||||
# Old behaviour: skip reason mentions the env var.
|
||||
assert "HERMES_DASHBOARD_OAUTH_CLIENT_ID" in nous_plugin.LAST_SKIP_REASON
|
||||
# New behaviour: skip reason ALSO mentions the config.yaml path
|
||||
# so the user knows it's a valid alternative.
|
||||
assert "dashboard.oauth.client_id" in nous_plugin.LAST_SKIP_REASON, (
|
||||
f"skip reason omits the config.yaml surface — operators "
|
||||
f"won't know it exists. got: {nous_plugin.LAST_SKIP_REASON!r}"
|
||||
)
|
||||
|
||||
def test_config_yaml_load_failure_falls_through_cleanly(
|
||||
self, monkeypatch
|
||||
):
|
||||
"""If load_config() raises (e.g. malformed YAML, IOError), the
|
||||
plugin must not crash — it falls through to the env-only path
|
||||
and either succeeds (if env is set) or surfaces the standard
|
||||
'not set' skip reason."""
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
|
||||
|
||||
def _broken_load():
|
||||
raise OSError("config.yaml not readable")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.load_config", _broken_load
|
||||
)
|
||||
ctx = MagicMock()
|
||||
# Must not raise.
|
||||
nous_plugin.register(ctx)
|
||||
ctx.register_dashboard_auth_provider.assert_not_called()
|
||||
|
||||
def test_config_yaml_with_non_dict_oauth_section(
|
||||
self, monkeypatch
|
||||
):
|
||||
"""cfg_get handles 'config has a string where a section was
|
||||
expected' robustly. Verify the plugin inherits that resilience
|
||||
so a malformed user config doesn't crash startup."""
|
||||
monkeypatch.delenv("HERMES_DASHBOARD_OAUTH_CLIENT_ID", raising=False)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.load_config",
|
||||
lambda: {"dashboard": {"oauth": "wrong type"}},
|
||||
)
|
||||
ctx = MagicMock()
|
||||
nous_plugin.register(ctx)
|
||||
# Falls through to the no-env-and-no-config path.
|
||||
ctx.register_dashboard_auth_provider.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# start_login
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStartLogin:
|
||||
@pytest.fixture
|
||||
def provider(self):
|
||||
return nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:inst1", portal_url="https://portal.example.com"
|
||||
)
|
||||
|
||||
def test_returns_login_start(self, provider):
|
||||
result = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
assert isinstance(result, LoginStart)
|
||||
|
||||
def test_redirect_url_targets_portal_authorize(self, provider):
|
||||
result = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
assert result.redirect_url.startswith(
|
||||
"https://portal.example.com/oauth/authorize?"
|
||||
)
|
||||
|
||||
def test_authorize_url_has_required_params(self, provider):
|
||||
result = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
parsed = urllib.parse.urlparse(result.redirect_url)
|
||||
params = dict(urllib.parse.parse_qsl(parsed.query))
|
||||
assert params["response_type"] == "code"
|
||||
assert params["client_id"] == "agent:inst1"
|
||||
assert params["redirect_uri"] == "https://hermes.fly.dev/auth/callback"
|
||||
assert params["scope"] == "agent_dashboard:access"
|
||||
assert params["code_challenge_method"] == "S256"
|
||||
assert "state" in params
|
||||
assert "code_challenge" in params
|
||||
|
||||
def test_code_verifier_in_cookie_payload_43_to_128_chars(self, provider):
|
||||
result = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
assert "hermes_session_pkce" in result.cookie_payload
|
||||
pkce = result.cookie_payload["hermes_session_pkce"]
|
||||
# Shape: ``state=…;verifier=…`` (matches stub-provider convention so
|
||||
# the auth-route layer's parser works uniformly across providers).
|
||||
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
|
||||
verifier = parts["verifier"]
|
||||
# RFC 7636 §4.1
|
||||
assert 43 <= len(verifier) <= 128
|
||||
|
||||
def test_state_in_cookie_payload_matches_url_param(self, provider):
|
||||
result = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
parsed = urllib.parse.urlparse(result.redirect_url)
|
||||
params = dict(urllib.parse.parse_qsl(parsed.query))
|
||||
pkce = result.cookie_payload["hermes_session_pkce"]
|
||||
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
|
||||
assert parts["state"] == params["state"]
|
||||
|
||||
def test_code_challenge_is_s256_of_verifier(self, provider):
|
||||
result = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
parsed = urllib.parse.urlparse(result.redirect_url)
|
||||
params = dict(urllib.parse.parse_qsl(parsed.query))
|
||||
pkce = result.cookie_payload["hermes_session_pkce"]
|
||||
parts = dict(seg.split("=", 1) for seg in pkce.split(";") if "=" in seg)
|
||||
verifier = parts["verifier"]
|
||||
expected_challenge = (
|
||||
base64.urlsafe_b64encode(
|
||||
hashlib.sha256(verifier.encode("ascii")).digest()
|
||||
)
|
||||
.rstrip(b"=")
|
||||
.decode()
|
||||
)
|
||||
assert params["code_challenge"] == expected_challenge
|
||||
|
||||
def test_two_calls_produce_different_state_and_verifier(self, provider):
|
||||
a = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
b = provider.start_login(
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
assert a.cookie_payload["hermes_session_pkce"] != b.cookie_payload[
|
||||
"hermes_session_pkce"
|
||||
]
|
||||
|
||||
def test_rejects_non_http_scheme(self, provider):
|
||||
with pytest.raises(ProviderError, match="http"):
|
||||
provider.start_login(redirect_uri="ftp://x/auth/callback")
|
||||
|
||||
def test_rejects_http_with_non_localhost(self, provider):
|
||||
with pytest.raises(ProviderError, match="localhost"):
|
||||
provider.start_login(
|
||||
redirect_uri="http://hermes.fly.dev/auth/callback"
|
||||
)
|
||||
|
||||
def test_allows_http_localhost(self, provider):
|
||||
# Should not raise.
|
||||
provider.start_login(redirect_uri="http://localhost:8080/auth/callback")
|
||||
provider.start_login(redirect_uri="http://127.0.0.1:8080/auth/callback")
|
||||
|
||||
def test_rejects_wrong_callback_path(self, provider):
|
||||
with pytest.raises(ProviderError, match="/auth/callback"):
|
||||
provider.start_login(redirect_uri="https://x.example/oauth/cb")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# complete_login (httpx mocked)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCompleteLogin:
|
||||
@pytest.fixture
|
||||
def provider(self, rsa_keypair):
|
||||
p = nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:inst123", portal_url="https://portal.example.com"
|
||||
)
|
||||
_patched_jwks(p, rsa_keypair)
|
||||
return p
|
||||
|
||||
def _mock_post(self, status_code: int, body: Any, *, ctype: str = "application/json"):
|
||||
resp = MagicMock(spec=httpx.Response)
|
||||
resp.status_code = status_code
|
||||
if isinstance(body, dict):
|
||||
resp.text = json.dumps(body)
|
||||
resp.json = MagicMock(return_value=body)
|
||||
else:
|
||||
resp.text = body
|
||||
# _parse_json_body bails on non-application/json before .json()
|
||||
# is called, but be safe for callers that pass a non-dict body
|
||||
# with ctype=application/json.
|
||||
resp.json = MagicMock(side_effect=ValueError("not json"))
|
||||
resp.headers = {"content-type": ctype}
|
||||
return resp
|
||||
|
||||
def test_happy_path_returns_session(self, provider, rsa_keypair):
|
||||
access_token = _mint_token(rsa_keypair)
|
||||
mock_resp = self._mock_post(
|
||||
200, {"access_token": access_token, "token_type": "Bearer"}
|
||||
)
|
||||
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
|
||||
session = provider.complete_login(
|
||||
code="abc",
|
||||
state="state-val",
|
||||
code_verifier="vfy",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
assert isinstance(session, Session)
|
||||
assert session.user_id == "usr_abc"
|
||||
assert session.provider == "nous"
|
||||
assert session.access_token == access_token
|
||||
assert session.refresh_token == "" # contract V1
|
||||
assert session.org_id == "org_xyz"
|
||||
assert session.email == ""
|
||||
assert session.display_name == ""
|
||||
|
||||
def test_400_raises_invalid_code(self, provider):
|
||||
mock_resp = self._mock_post(400, {"error": "invalid_grant"})
|
||||
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
|
||||
with pytest.raises(InvalidCodeError, match="invalid_grant"):
|
||||
provider.complete_login(
|
||||
code="bad", state="s", code_verifier="v",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
|
||||
def test_500_raises_provider_error(self, provider):
|
||||
mock_resp = self._mock_post(500, "internal server error", ctype="text/plain")
|
||||
mock_resp.text = "internal server error"
|
||||
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
|
||||
with pytest.raises(ProviderError, match="500"):
|
||||
provider.complete_login(
|
||||
code="x", state="s", code_verifier="v",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
|
||||
def test_missing_access_token_raises(self, provider):
|
||||
mock_resp = self._mock_post(200, {"token_type": "Bearer"})
|
||||
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
|
||||
with pytest.raises(ProviderError, match="access_token"):
|
||||
provider.complete_login(
|
||||
code="x", state="s", code_verifier="v",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
|
||||
def test_unexpected_token_type_raises(self, provider, rsa_keypair):
|
||||
access_token = _mint_token(rsa_keypair)
|
||||
mock_resp = self._mock_post(
|
||||
200, {"access_token": access_token, "token_type": "DPoP"}
|
||||
)
|
||||
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
|
||||
with pytest.raises(ProviderError, match="token_type"):
|
||||
provider.complete_login(
|
||||
code="x", state="s", code_verifier="v",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
|
||||
def test_network_error_raises_provider_error(self, provider):
|
||||
with patch(
|
||||
"plugins.dashboard_auth.nous.httpx.post",
|
||||
side_effect=httpx.ConnectError("conn refused"),
|
||||
):
|
||||
with pytest.raises(ProviderError, match="unreachable"):
|
||||
provider.complete_login(
|
||||
code="x", state="s", code_verifier="v",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
|
||||
def test_captures_refresh_token_if_present_forward_compat(
|
||||
self, provider, rsa_keypair
|
||||
):
|
||||
"""Forward-compat: contract V1 doesn't issue, but if a future Portal
|
||||
does, we should preserve it in the Session for later use."""
|
||||
access_token = _mint_token(rsa_keypair)
|
||||
mock_resp = self._mock_post(
|
||||
200,
|
||||
{
|
||||
"access_token": access_token,
|
||||
"token_type": "Bearer",
|
||||
"refresh_token": "rt-opaque",
|
||||
},
|
||||
)
|
||||
with patch("plugins.dashboard_auth.nous.httpx.post", return_value=mock_resp):
|
||||
session = provider.complete_login(
|
||||
code="x", state="s", code_verifier="v",
|
||||
redirect_uri="https://hermes.fly.dev/auth/callback",
|
||||
)
|
||||
assert session.refresh_token == "rt-opaque"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# verify_session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifySession:
|
||||
@pytest.fixture
|
||||
def provider(self, rsa_keypair):
|
||||
p = nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:inst123", portal_url="https://portal.example.com"
|
||||
)
|
||||
_patched_jwks(p, rsa_keypair)
|
||||
return p
|
||||
|
||||
def test_happy_path_returns_session(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair)
|
||||
session = provider.verify_session(access_token=token)
|
||||
assert session is not None
|
||||
assert session.user_id == "usr_abc"
|
||||
assert session.org_id == "org_xyz"
|
||||
|
||||
def test_expired_token_returns_none(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair, ttl_seconds=-1)
|
||||
assert provider.verify_session(access_token=token) is None
|
||||
|
||||
def test_wrong_audience_raises_provider_error(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair, aud="agent:other-instance")
|
||||
with pytest.raises(ProviderError, match="verification failed"):
|
||||
provider.verify_session(access_token=token)
|
||||
|
||||
def test_wrong_issuer_raises_provider_error(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair, iss="https://evil.example")
|
||||
with pytest.raises(ProviderError, match="verification failed"):
|
||||
provider.verify_session(access_token=token)
|
||||
|
||||
def test_verification_failure_message_surfaces_token_claims(
|
||||
self, provider, rsa_keypair
|
||||
):
|
||||
"""Operators need to see the actual iss/aud the token carries to debug
|
||||
config drift between HERMES_DASHBOARD_PORTAL_URL/CLIENT_ID and Portal."""
|
||||
token = _mint_token(rsa_keypair, iss="https://evil.example")
|
||||
with pytest.raises(ProviderError) as excinfo:
|
||||
provider.verify_session(access_token=token)
|
||||
msg = str(excinfo.value)
|
||||
# Both the observed (token) and expected (configured) values appear.
|
||||
assert "'https://evil.example'" in msg
|
||||
assert "'https://portal.example.com'" in msg # configured portal URL
|
||||
|
||||
def test_missing_sub_raises(self, provider, rsa_keypair):
|
||||
# PyJWT's "require" set includes sub, so this surfaces as
|
||||
# InvalidTokenError → ProviderError before we ever touch _session_from_claims.
|
||||
token = _mint_token(rsa_keypair, sub="")
|
||||
# Empty sub still encodes successfully; PyJWT's require check only
|
||||
# asserts presence. Our own _session_from_claims rejects empty.
|
||||
with pytest.raises(ProviderError, match="sub"):
|
||||
provider.verify_session(access_token=token)
|
||||
|
||||
def test_agent_instance_id_mismatch_rejected(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair, agent_instance_id="some-other-id")
|
||||
with pytest.raises(ProviderError, match="agent_instance_id mismatch"):
|
||||
provider.verify_session(access_token=token)
|
||||
|
||||
def test_agent_instance_id_missing_is_tolerated(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair, agent_instance_id=None)
|
||||
session = provider.verify_session(access_token=token)
|
||||
assert session is not None
|
||||
|
||||
def test_contract_version_missing_warns_but_succeeds(
|
||||
self, provider, rsa_keypair, caplog
|
||||
):
|
||||
import logging
|
||||
token = _mint_token(rsa_keypair, oauth_contract_version=None)
|
||||
with caplog.at_level(logging.WARNING, logger="plugins.dashboard_auth.nous"):
|
||||
session = provider.verify_session(access_token=token)
|
||||
assert session is not None
|
||||
assert any(
|
||||
"oauth_contract_version" in r.message for r in caplog.records
|
||||
)
|
||||
|
||||
def test_contract_version_mismatch_rejected(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair, oauth_contract_version=2)
|
||||
with pytest.raises(ProviderError, match="oauth_contract_version"):
|
||||
provider.verify_session(access_token=token)
|
||||
|
||||
def test_jwks_unreachable_raises_provider_error(self, provider, rsa_keypair):
|
||||
token = _mint_token(rsa_keypair)
|
||||
# Replace the patched client so it raises.
|
||||
bad_client = MagicMock()
|
||||
bad_client.get_signing_key_from_jwt.side_effect = jwt.PyJWKClientError(
|
||||
"fetch failed"
|
||||
)
|
||||
provider._jwks_client = bad_client
|
||||
with pytest.raises(ProviderError, match="JWKS"):
|
||||
provider.verify_session(access_token=token)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# refresh_session + revoke_session (V1 contract: trivial)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRefreshAndRevoke:
|
||||
@pytest.fixture
|
||||
def provider(self):
|
||||
return nous_plugin.NousDashboardAuthProvider(
|
||||
client_id="agent:inst1", portal_url="https://portal.example.com"
|
||||
)
|
||||
|
||||
def test_refresh_always_raises(self, provider):
|
||||
with pytest.raises(RefreshExpiredError):
|
||||
provider.refresh_session(refresh_token="anything")
|
||||
|
||||
def test_refresh_raises_even_with_empty_token(self, provider):
|
||||
with pytest.raises(RefreshExpiredError):
|
||||
provider.refresh_session(refresh_token="")
|
||||
|
||||
def test_revoke_is_noop(self, provider):
|
||||
# Must not raise; returns None implicitly.
|
||||
assert provider.revoke_session(refresh_token="anything") is None
|
||||
assert provider.revoke_session(refresh_token="") is None
|
||||
@@ -0,0 +1,300 @@
|
||||
"""Behavior-parity check for the image-gen FAL plugin migration (#26241).
|
||||
|
||||
Spawns one subprocess per (version, scenario) cell — pinned to either
|
||||
``origin/main`` (legacy in-tree FAL fall-through + ``configured == "fal"``
|
||||
skip in ``_dispatch_to_plugin_provider``) or this PR's worktree (FAL is
|
||||
itself a plugin and the dispatcher routes every set provider through
|
||||
the registry). Each subprocess clears all FAL-related env vars + writes
|
||||
a ``config.yaml``, then asks the dispatcher how it would route an
|
||||
``image_generate`` call. The emitted shape tuple is
|
||||
``{dispatch_kind, provider_name, model}``:
|
||||
|
||||
* ``dispatch_kind`` ∈ ``{"legacy_fal", "plugin", "error", None}`` —
|
||||
whether the call would go straight to the in-tree pipeline,
|
||||
through ``_dispatch_to_plugin_provider``, raise an explicit
|
||||
provider-not-registered error, or fall through silently.
|
||||
* ``provider_name`` — when ``dispatch_kind == "plugin"``, the
|
||||
resolved provider name. ``None`` otherwise.
|
||||
* ``model`` — the resolved FAL model id when applicable.
|
||||
|
||||
The parent process diffs the shapes per scenario. A diff means the
|
||||
migration introduced an observable behaviour change vs origin/main —
|
||||
likely a real regression for users on the existing config keys.
|
||||
|
||||
Run from the PR worktree:
|
||||
|
||||
python tests/plugins/image_gen/check_parity_vs_main.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
# Pin one path to current main, one to the PR worktree.
|
||||
# ``REPO_ROOT`` is ``.../.worktrees/<name>``; the main checkout lives
|
||||
# two levels up. When running directly from a regular clone (no
|
||||
# worktree), ``MAIN_DIR`` falls back to a sibling ``hermes-agent-main``
|
||||
# checkout if one exists.
|
||||
def _resolve_main_dir() -> Path:
|
||||
candidate = REPO_ROOT.parent.parent
|
||||
if (candidate / "tools" / "image_generation_tool.py").exists() and candidate != REPO_ROOT:
|
||||
return candidate
|
||||
sibling = REPO_ROOT.parent / "hermes-agent-main"
|
||||
if (sibling / "tools" / "image_generation_tool.py").exists():
|
||||
return sibling
|
||||
return REPO_ROOT
|
||||
|
||||
|
||||
MAIN_DIR = _resolve_main_dir()
|
||||
PR_DIR = REPO_ROOT
|
||||
assert (PR_DIR / "tools" / "image_generation_tool.py").exists(), (
|
||||
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
|
||||
)
|
||||
|
||||
|
||||
SUBPROCESS_SCRIPT = r"""
|
||||
import json, os, sys, tempfile
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
|
||||
# Isolated HERMES_HOME so the config write is hermetic.
|
||||
home = tempfile.mkdtemp()
|
||||
os.environ["HERMES_HOME"] = home
|
||||
|
||||
# Clear FAL-related env so dispatch decisions are config-driven.
|
||||
for k in (
|
||||
"FAL_KEY", "FAL_QUEUE_GATEWAY_URL",
|
||||
"TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN",
|
||||
"FAL_IMAGE_MODEL",
|
||||
):
|
||||
os.environ.pop(k, None)
|
||||
|
||||
scenario_env = json.loads(sys.argv[2])
|
||||
os.environ.update(scenario_env)
|
||||
|
||||
config_yaml = sys.argv[3]
|
||||
config_path = os.path.join(home, "config.yaml")
|
||||
with open(config_path, "w") as f:
|
||||
f.write(config_yaml)
|
||||
|
||||
# Fresh import — must not have anything cached.
|
||||
for name in list(sys.modules):
|
||||
if (name.startswith("tools.")
|
||||
or name.startswith("agent.")
|
||||
or name.startswith("plugins.")
|
||||
or name.startswith("hermes_cli.")):
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
import tools.image_generation_tool as image_tool
|
||||
|
||||
dispatch_kind = None
|
||||
provider_name = None
|
||||
model = None
|
||||
error_text = None
|
||||
|
||||
try:
|
||||
raw = image_tool._dispatch_to_plugin_provider("ping", "landscape")
|
||||
if raw is None:
|
||||
dispatch_kind = "legacy_fal"
|
||||
else:
|
||||
parsed = json.loads(raw) if isinstance(raw, str) else raw
|
||||
if isinstance(parsed, dict):
|
||||
if parsed.get("error_type") == "provider_not_registered":
|
||||
dispatch_kind = "error"
|
||||
error_text = parsed.get("error")
|
||||
else:
|
||||
dispatch_kind = "plugin"
|
||||
provider_name = parsed.get("provider")
|
||||
model = parsed.get("model")
|
||||
else:
|
||||
dispatch_kind = "unknown_payload"
|
||||
|
||||
if model is None:
|
||||
# _resolve_fal_model still returns the active FAL model id even
|
||||
# when dispatch goes to a non-FAL plugin — used for the diff
|
||||
# only when applicable.
|
||||
try:
|
||||
model_id, _meta = image_tool._resolve_fal_model()
|
||||
if dispatch_kind == "legacy_fal":
|
||||
model = model_id
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
dispatch_kind = "exception"
|
||||
error_text = repr(exc)
|
||||
|
||||
shape = {
|
||||
"dispatch_kind": dispatch_kind,
|
||||
"provider_name": provider_name,
|
||||
"model": model,
|
||||
"error_present": error_text is not None,
|
||||
}
|
||||
print(json.dumps(shape))
|
||||
"""
|
||||
|
||||
|
||||
SCENARIOS: list[tuple[str, str, dict[str, str]]] = [
|
||||
# (label, config.yaml body, extra env vars)
|
||||
("no-config-no-env", "", {}),
|
||||
(
|
||||
"explicit-fal-no-creds",
|
||||
"image_gen:\n provider: fal\n",
|
||||
{},
|
||||
),
|
||||
(
|
||||
"explicit-fal-with-creds",
|
||||
"image_gen:\n provider: fal\n",
|
||||
{"FAL_KEY": "test-key"},
|
||||
),
|
||||
(
|
||||
"explicit-fal-with-model",
|
||||
"image_gen:\n provider: fal\n model: fal-ai/flux-2-pro\n",
|
||||
{"FAL_KEY": "test-key"},
|
||||
),
|
||||
(
|
||||
"explicit-typo-provider",
|
||||
"image_gen:\n provider: not-a-real-backend\n",
|
||||
{"FAL_KEY": "test-key"},
|
||||
),
|
||||
(
|
||||
"managed-gateway-only",
|
||||
"",
|
||||
{
|
||||
"TOOL_GATEWAY_DOMAIN": "nousresearch.com",
|
||||
"TOOL_GATEWAY_USER_TOKEN": "nous-token",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict) -> dict:
|
||||
venv_python = repo_path / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = Path("python3")
|
||||
|
||||
out = subprocess.run(
|
||||
[
|
||||
str(venv_python),
|
||||
"-c",
|
||||
SUBPROCESS_SCRIPT,
|
||||
str(repo_path),
|
||||
json.dumps(env),
|
||||
config_yaml,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return {
|
||||
"error": "subprocess failed",
|
||||
"stdout": out.stdout[-500:],
|
||||
"stderr": out.stderr[-500:],
|
||||
}
|
||||
try:
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
except Exception as exc:
|
||||
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
|
||||
|
||||
|
||||
def _reduce(shape: dict) -> dict:
|
||||
"""Reduce to the parts that matter for user-visible parity.
|
||||
|
||||
On origin/main, ``explicit-fal-*`` scenarios short-circuit to
|
||||
``legacy_fal`` because of the ``configured == "fal"`` skip. On the
|
||||
PR, those same scenarios route through the plugin and emit
|
||||
``dispatch_kind == "plugin"`` with ``provider_name == "fal"``.
|
||||
|
||||
Both shapes are functionally equivalent — the plugin's ``generate()``
|
||||
re-enters the same in-tree pipeline via ``_it`` indirection — but
|
||||
we want the diff to be visible so reviewers can sign off on the
|
||||
intentional behaviour delta.
|
||||
"""
|
||||
return {
|
||||
"dispatch_kind": shape.get("dispatch_kind"),
|
||||
"provider_name": shape.get("provider_name"),
|
||||
"model": shape.get("model"),
|
||||
"error_present": shape.get("error_present"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print(f"main: {MAIN_DIR}")
|
||||
print(f"pr: {PR_DIR}")
|
||||
print()
|
||||
|
||||
if MAIN_DIR == PR_DIR:
|
||||
print(
|
||||
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
|
||||
" Set up a sibling 'hermes-agent-main' checkout pinned to "
|
||||
"origin/main to get real parity coverage."
|
||||
)
|
||||
print()
|
||||
|
||||
failures: list[str] = []
|
||||
errors: list[str] = []
|
||||
intentional_diffs: list[tuple[str, dict, dict]] = []
|
||||
for label, config_yaml, env in SCENARIOS:
|
||||
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env)
|
||||
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env)
|
||||
|
||||
if "error" in main_shape or "error" in pr_shape:
|
||||
print(f" [ERR ] {label}: subprocess failed")
|
||||
print(f" main: {main_shape}")
|
||||
print(f" pr: {pr_shape}")
|
||||
errors.append(label)
|
||||
continue
|
||||
|
||||
main_reduced = _reduce(main_shape)
|
||||
pr_reduced = _reduce(pr_shape)
|
||||
|
||||
if main_reduced == pr_reduced:
|
||||
print(f" [OK] {label}: {main_reduced}")
|
||||
continue
|
||||
|
||||
# On main, "explicit-fal-*" returns legacy_fal; on PR, plugin
|
||||
# dispatch. That's the only acceptable diff — flag everything
|
||||
# else as a regression.
|
||||
legacy_to_plugin_fal = (
|
||||
main_reduced.get("dispatch_kind") == "legacy_fal"
|
||||
and pr_reduced.get("dispatch_kind") == "plugin"
|
||||
and pr_reduced.get("provider_name") == "fal"
|
||||
)
|
||||
if legacy_to_plugin_fal:
|
||||
print(f" [DIFF] {label}: legacy_fal → plugin (fal) — expected")
|
||||
intentional_diffs.append((label, main_reduced, pr_reduced))
|
||||
else:
|
||||
print(f" [FAIL] {label}")
|
||||
print(f" main: {main_reduced}")
|
||||
print(f" pr: {pr_reduced}")
|
||||
failures.append(label)
|
||||
|
||||
print()
|
||||
if errors:
|
||||
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
if failures:
|
||||
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
|
||||
for f in failures:
|
||||
print(f" - {f}")
|
||||
if intentional_diffs:
|
||||
print(
|
||||
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
|
||||
f"legacy_fal → plugin dispatch for explicit FAL paths."
|
||||
)
|
||||
if failures or errors:
|
||||
return 1
|
||||
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for the FAL.ai image generation plugin.
|
||||
|
||||
The plugin is a thin registration adapter — actual FAL pipeline logic
|
||||
lives in ``tools.image_generation_tool`` and is exercised by
|
||||
``tests/tools/test_image_generation.py``. These tests focus on:
|
||||
|
||||
* the ``ImageGenProvider`` ABC surface (name, models, schema)
|
||||
* call-time indirection (``_it`` resolution at ``generate()`` time so
|
||||
``monkeypatch.setattr(image_tool, ...)`` keeps working)
|
||||
* response shape stamping (provider/prompt/aspect_ratio/model)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider surface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFalImageGenProviderSurface:
|
||||
def test_name(self):
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
assert FalImageGenProvider().name == "fal"
|
||||
|
||||
def test_display_name(self):
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
assert FalImageGenProvider().display_name == "FAL.ai"
|
||||
|
||||
def test_default_model_matches_legacy(self):
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
from tools.image_generation_tool import DEFAULT_MODEL
|
||||
|
||||
assert FalImageGenProvider().default_model() == DEFAULT_MODEL
|
||||
|
||||
def test_list_models_uses_legacy_catalog(self):
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
from tools.image_generation_tool import FAL_MODELS
|
||||
|
||||
provider = FalImageGenProvider()
|
||||
models = provider.list_models()
|
||||
ids = {m["id"] for m in models}
|
||||
# Whatever FAL_MODELS ships, the provider mirrors verbatim.
|
||||
assert ids == set(FAL_MODELS.keys())
|
||||
# Spot-check the expected first-class fields are present.
|
||||
for entry in models:
|
||||
for field in ("id", "display", "speed", "strengths", "price"):
|
||||
assert field in entry
|
||||
|
||||
def test_setup_schema_advertises_fal_key(self):
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
schema = FalImageGenProvider().get_setup_schema()
|
||||
assert schema["name"] == "FAL.ai"
|
||||
assert schema["badge"] == "paid"
|
||||
env_keys = {entry["key"] for entry in schema.get("env_vars", [])}
|
||||
assert "FAL_KEY" in env_keys
|
||||
|
||||
|
||||
class TestFalImageGenProviderAvailability:
|
||||
def test_is_available_when_legacy_check_passes(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: True)
|
||||
assert FalImageGenProvider().is_available() is True
|
||||
|
||||
def test_is_available_false_when_legacy_check_fails(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
monkeypatch.setattr(image_tool, "check_fal_api_key", lambda: False)
|
||||
assert FalImageGenProvider().is_available() is False
|
||||
|
||||
def test_is_available_handles_legacy_exception(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("config broke")
|
||||
|
||||
monkeypatch.setattr(image_tool, "check_fal_api_key", _boom)
|
||||
# Picker must not propagate exceptions — show as "not available".
|
||||
assert FalImageGenProvider().is_available() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# generate() — call-time indirection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFalImageGenProviderGenerate:
|
||||
def test_generate_delegates_to_legacy_image_generate_tool(self, monkeypatch):
|
||||
"""Plugin must look up ``image_generate_tool`` at call time so
|
||||
``monkeypatch.setattr(image_tool, "image_generate_tool", ...)``
|
||||
takes effect."""
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_image_generate_tool(prompt, aspect_ratio, **kwargs):
|
||||
captured["prompt"] = prompt
|
||||
captured["aspect_ratio"] = aspect_ratio
|
||||
captured["kwargs"] = kwargs
|
||||
return json.dumps({"success": True, "image": "https://fake/image.png"})
|
||||
|
||||
monkeypatch.setattr(image_tool, "image_generate_tool", fake_image_generate_tool)
|
||||
monkeypatch.setattr(image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2/klein/9b", {}))
|
||||
|
||||
result = FalImageGenProvider().generate(
|
||||
"a serene mountain landscape",
|
||||
aspect_ratio="square",
|
||||
seed=42,
|
||||
)
|
||||
|
||||
assert captured["prompt"] == "a serene mountain landscape"
|
||||
assert captured["aspect_ratio"] == "square"
|
||||
assert captured["kwargs"] == {"seed": 42}
|
||||
assert result["success"] is True
|
||||
assert result["image"] == "https://fake/image.png"
|
||||
# Stamped fields for the unified response shape
|
||||
assert result["provider"] == "fal"
|
||||
assert result["prompt"] == "a serene mountain landscape"
|
||||
assert result["aspect_ratio"] == "square"
|
||||
assert result["model"] == "fal-ai/flux-2/klein/9b"
|
||||
|
||||
def test_generate_invalid_aspect_ratio_is_coerced(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
seen_aspect = {}
|
||||
|
||||
def fake(prompt, aspect_ratio, **kwargs):
|
||||
seen_aspect["v"] = aspect_ratio
|
||||
return json.dumps({"success": True, "image": "x"})
|
||||
|
||||
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
|
||||
monkeypatch.setattr(image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2/klein/9b", {}))
|
||||
|
||||
FalImageGenProvider().generate("p", aspect_ratio="not-a-real-ratio")
|
||||
# ``resolve_aspect_ratio`` clamps to landscape.
|
||||
assert seen_aspect["v"] == "landscape"
|
||||
|
||||
def test_generate_passthrough_drops_none_kwargs(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
seen = {}
|
||||
|
||||
def fake(prompt, aspect_ratio, **kwargs):
|
||||
seen.update(kwargs)
|
||||
return json.dumps({"success": True, "image": "x"})
|
||||
|
||||
monkeypatch.setattr(image_tool, "image_generate_tool", fake)
|
||||
monkeypatch.setattr(image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2/klein/9b", {}))
|
||||
|
||||
FalImageGenProvider().generate(
|
||||
"p",
|
||||
aspect_ratio="landscape",
|
||||
seed=None,
|
||||
num_images=2,
|
||||
guidance_scale=None,
|
||||
)
|
||||
|
||||
# ``None`` values must not be forwarded — they'd override the
|
||||
# model's defaults inside the legacy payload builder.
|
||||
assert "seed" not in seen
|
||||
assert "guidance_scale" not in seen
|
||||
assert seen.get("num_images") == 2
|
||||
|
||||
def test_generate_catches_exception_from_legacy(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError("FAL endpoint exploded")
|
||||
|
||||
monkeypatch.setattr(image_tool, "image_generate_tool", boom)
|
||||
|
||||
result = FalImageGenProvider().generate("p")
|
||||
assert result["success"] is False
|
||||
assert "FAL image generation failed" in result["error"]
|
||||
assert result["error_type"] == "RuntimeError"
|
||||
assert result["provider"] == "fal"
|
||||
|
||||
def test_generate_invalid_json_response(self, monkeypatch):
|
||||
import tools.image_generation_tool as image_tool
|
||||
from plugins.image_gen.fal import FalImageGenProvider
|
||||
|
||||
monkeypatch.setattr(image_tool, "image_generate_tool", lambda **kw: "not-json")
|
||||
monkeypatch.setattr(image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2/klein/9b", {}))
|
||||
|
||||
result = FalImageGenProvider().generate("p")
|
||||
assert result["success"] is False
|
||||
assert "Invalid JSON" in result["error"]
|
||||
assert result["provider"] == "fal"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFalImageGenPluginRegistration:
|
||||
def test_register_wires_provider_into_registry(self):
|
||||
from plugins.image_gen.fal import FalImageGenProvider, register
|
||||
|
||||
ctx = MagicMock()
|
||||
register(ctx)
|
||||
|
||||
ctx.register_image_gen_provider.assert_called_once()
|
||||
(registered,), _ = ctx.register_image_gen_provider.call_args
|
||||
assert isinstance(registered, FalImageGenProvider)
|
||||
@@ -0,0 +1,625 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for Krea image generation provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fake_api_key(monkeypatch):
|
||||
"""Ensure KREA_API_KEY is set for all tests."""
|
||||
monkeypatch.setenv("KREA_API_KEY", "test-key-12345")
|
||||
|
||||
|
||||
def _completed_job(url: str = "https://krea.cdn/img.png") -> dict:
|
||||
return {
|
||||
"job_id": "00000000-0000-0000-0000-000000000abc",
|
||||
"status": "completed",
|
||||
"created_at": "2026-05-27T00:00:00Z",
|
||||
"completed_at": "2026-05-27T00:00:30Z",
|
||||
"result": {"urls": [url]},
|
||||
}
|
||||
|
||||
|
||||
def _submit_response(job_id: str = "00000000-0000-0000-0000-000000000abc"):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.raise_for_status = MagicMock()
|
||||
resp.json.return_value = {
|
||||
"job_id": job_id,
|
||||
"status": "queued",
|
||||
"created_at": "2026-05-27T00:00:00Z",
|
||||
"completed_at": None,
|
||||
"result": None,
|
||||
}
|
||||
return resp
|
||||
|
||||
|
||||
def _poll_response(body: dict):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.raise_for_status = MagicMock()
|
||||
resp.json.return_value = body
|
||||
return resp
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider class tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestKreaImageGenProvider:
|
||||
def test_name(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
assert KreaImageGenProvider().name == "krea"
|
||||
|
||||
def test_display_name(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
assert KreaImageGenProvider().display_name == "Krea"
|
||||
|
||||
def test_is_available_with_key(self, monkeypatch):
|
||||
monkeypatch.setenv("KREA_API_KEY", "sk-test")
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
assert KreaImageGenProvider().is_available() is True
|
||||
|
||||
def test_is_available_without_key(self, monkeypatch):
|
||||
monkeypatch.delenv("KREA_API_KEY", raising=False)
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
assert KreaImageGenProvider().is_available() is False
|
||||
|
||||
def test_list_models(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
models = KreaImageGenProvider().list_models()
|
||||
ids = {m["id"] for m in models}
|
||||
assert {"krea-2-medium", "krea-2-large"} <= ids
|
||||
# Each entry carries the picker fields the registry expects.
|
||||
for m in models:
|
||||
assert m["display"]
|
||||
assert m["speed"]
|
||||
assert m["strengths"]
|
||||
assert m["price"]
|
||||
|
||||
def test_default_model_is_medium(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
assert KreaImageGenProvider().default_model() == "krea-2-medium"
|
||||
|
||||
def test_get_setup_schema(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
schema = KreaImageGenProvider().get_setup_schema()
|
||||
assert schema["name"] == "Krea"
|
||||
assert schema["badge"] == "paid"
|
||||
env_vars = schema["env_vars"]
|
||||
assert len(env_vars) == 1
|
||||
assert env_vars[0]["key"] == "KREA_API_KEY"
|
||||
assert "krea.ai" in env_vars[0]["url"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestModelResolution:
|
||||
def test_default(self):
|
||||
from plugins.image_gen.krea import _resolve_model
|
||||
|
||||
model_id, meta = _resolve_model()
|
||||
assert model_id == "krea-2-medium"
|
||||
assert meta["path"] == "medium"
|
||||
|
||||
def test_env_override_large(self, monkeypatch):
|
||||
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
|
||||
from plugins.image_gen.krea import _resolve_model
|
||||
|
||||
model_id, meta = _resolve_model()
|
||||
assert model_id == "krea-2-large"
|
||||
assert meta["path"] == "large"
|
||||
|
||||
def test_env_override_unknown_falls_back_to_default(self, monkeypatch):
|
||||
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-xxl-fake")
|
||||
from plugins.image_gen.krea import _resolve_model
|
||||
|
||||
model_id, _ = _resolve_model()
|
||||
assert model_id == "krea-2-medium"
|
||||
|
||||
def test_creativity_default(self):
|
||||
from plugins.image_gen.krea import _resolve_creativity
|
||||
|
||||
assert _resolve_creativity(None) == "medium"
|
||||
|
||||
def test_creativity_valid(self):
|
||||
from plugins.image_gen.krea import _resolve_creativity
|
||||
|
||||
assert _resolve_creativity("HIGH") == "high"
|
||||
assert _resolve_creativity(" raw ") == "raw"
|
||||
|
||||
def test_creativity_invalid(self):
|
||||
from plugins.image_gen.krea import _resolve_creativity
|
||||
|
||||
assert _resolve_creativity("ultra") == "medium"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generate — main flow
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerate:
|
||||
def test_missing_api_key(self, monkeypatch):
|
||||
monkeypatch.delenv("KREA_API_KEY", raising=False)
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
assert result["success"] is False
|
||||
assert "KREA_API_KEY" in result["error"]
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
def test_empty_prompt(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
result = KreaImageGenProvider().generate(prompt=" ")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "invalid_argument"
|
||||
|
||||
def test_successful_generation(self):
|
||||
"""Happy path: submit → one poll → completed → URL downloaded."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job("https://krea.cdn/result.png"))
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll) as mock_get, \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/krea_krea-2-medium_test.png"),
|
||||
) as mock_save, \
|
||||
patch("plugins.image_gen.krea.time.sleep"): # skip real waits
|
||||
result = KreaImageGenProvider().generate(prompt="A cinematic lamp")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image"] == "/tmp/krea_krea-2-medium_test.png"
|
||||
assert result["provider"] == "krea"
|
||||
assert result["model"] == "krea-2-medium"
|
||||
assert result["aspect_ratio"] == "landscape"
|
||||
assert result["job_id"] == "00000000-0000-0000-0000-000000000abc"
|
||||
assert result["resolution"] == "1K"
|
||||
assert result["creativity"] == "medium"
|
||||
# Submit hit the medium endpoint
|
||||
post_url = mock_post.call_args[0][0]
|
||||
assert post_url.endswith("/generate/image/krea/krea-2/medium")
|
||||
# Poll hit /jobs/{job_id}
|
||||
poll_url = mock_get.call_args[0][0]
|
||||
assert "/jobs/00000000-0000-0000-0000-000000000abc" in poll_url
|
||||
# URL was materialised once
|
||||
mock_save.assert_called_once()
|
||||
|
||||
def test_large_model_routes_to_large_endpoint(self, monkeypatch):
|
||||
monkeypatch.setenv("KREA_IMAGE_MODEL", "krea-2-large")
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job())
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
post_url = mock_post.call_args[0][0]
|
||||
assert post_url.endswith("/generate/image/krea/krea-2/large")
|
||||
|
||||
def test_aspect_ratio_mapping(self):
|
||||
"""Hermes 'square' must map to Krea '1:1' in the wire payload."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job())
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
KreaImageGenProvider().generate(prompt="test", aspect_ratio="square")
|
||||
|
||||
payload = mock_post.call_args.kwargs["json"]
|
||||
assert payload["aspect_ratio"] == "1:1"
|
||||
assert payload["resolution"] == "1K"
|
||||
|
||||
def test_auth_header(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job())
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
headers = mock_post.call_args.kwargs["headers"]
|
||||
assert headers["Authorization"] == "Bearer test-key-12345"
|
||||
assert headers["Content-Type"] == "application/json"
|
||||
|
||||
def test_passthrough_seed_styles_moodboards(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job())
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit) as mock_post, \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
KreaImageGenProvider().generate(
|
||||
prompt="test",
|
||||
seed=42,
|
||||
styles=[{"id": "lora-1", "strength": 0.7}],
|
||||
moodboards=[{"url": "https://x.com/mood.png"}, {"url": "https://x.com/mood2.png"}],
|
||||
image_style_references=[{"url": f"https://x.com/{i}.png"} for i in range(15)],
|
||||
creativity="high",
|
||||
)
|
||||
|
||||
payload = mock_post.call_args.kwargs["json"]
|
||||
assert payload["seed"] == 42
|
||||
assert payload["styles"] == [{"id": "lora-1", "strength": 0.7}]
|
||||
assert len(payload["moodboards"]) == 1 # capped at 1
|
||||
assert len(payload["image_style_references"]) == 10 # capped at 10
|
||||
assert payload["creativity"] == "high"
|
||||
|
||||
def test_unknown_kwargs_ignored(self):
|
||||
"""Forward-compat: unknown kwargs must not break generate()."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job())
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(
|
||||
prompt="test",
|
||||
fictional_param="should be ignored",
|
||||
num_images=4,
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generate — error paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerateErrors:
|
||||
def test_submit_http_error(self):
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
resp = req_lib.Response()
|
||||
resp.status_code = 401
|
||||
resp._content = b'{"error": {"message": "Invalid API key"}}'
|
||||
resp.headers["Content-Type"] = "application/json"
|
||||
resp.raise_for_status = MagicMock(
|
||||
side_effect=req_lib.HTTPError(response=resp)
|
||||
)
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=resp):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "401" in result["error"]
|
||||
assert "Invalid API key" in result["error"]
|
||||
|
||||
def test_submit_timeout(self):
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
with patch(
|
||||
"plugins.image_gen.krea.requests.post", side_effect=req_lib.Timeout()
|
||||
):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "timeout"
|
||||
|
||||
def test_submit_connection_error(self):
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
with patch(
|
||||
"plugins.image_gen.krea.requests.post",
|
||||
side_effect=req_lib.ConnectionError("dns nope"),
|
||||
):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "connection_error"
|
||||
|
||||
def test_submit_missing_job_id(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
bad_submit = MagicMock()
|
||||
bad_submit.status_code = 200
|
||||
bad_submit.raise_for_status = MagicMock()
|
||||
bad_submit.json.return_value = {"status": "queued"}
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=bad_submit):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "invalid_response"
|
||||
assert "job_id" in result["error"]
|
||||
|
||||
def test_job_failed(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
failed = {
|
||||
"job_id": "abc",
|
||||
"status": "failed",
|
||||
"completed_at": "2026-05-27T00:01:00Z",
|
||||
"result": {"error": "NSFW content"},
|
||||
}
|
||||
|
||||
submit = _submit_response()
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.requests.get",
|
||||
return_value=_poll_response(failed),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "NSFW" in result["error"]
|
||||
|
||||
def test_job_cancelled(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
cancelled = {
|
||||
"job_id": "abc",
|
||||
"status": "cancelled",
|
||||
"completed_at": "2026-05-27T00:01:00Z",
|
||||
"result": {},
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.requests.get",
|
||||
return_value=_poll_response(cancelled),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "cancelled"
|
||||
|
||||
def test_completed_but_missing_urls(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
completed_empty = {
|
||||
"job_id": "abc",
|
||||
"status": "completed",
|
||||
"completed_at": "2026-05-27T00:01:00Z",
|
||||
"result": {"urls": []},
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.requests.get",
|
||||
return_value=_poll_response(completed_empty),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "empty_response"
|
||||
|
||||
def test_url_download_failure_falls_back_to_bare_url(self):
|
||||
"""Mirror of xAI behaviour — if local cache fails, return the URL."""
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
url = "https://krea.cdn/expired-soon.png"
|
||||
submit = _submit_response()
|
||||
poll = _poll_response(_completed_job(url))
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=submit), \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=poll), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
side_effect=req_lib.HTTPError("404"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image"] == url
|
||||
|
||||
def test_polling_picks_up_completed_at_with_unknown_status(self):
|
||||
"""``completed_at`` set + unrecognised pending status → still terminal."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
# Use a status value that is NOT in our terminal set ("intermediate-complete")
|
||||
# but with completed_at populated — Krea's spec says completed_at is the
|
||||
# canonical terminal marker.
|
||||
oddball = {
|
||||
"job_id": "abc",
|
||||
"status": "intermediate-complete",
|
||||
"completed_at": "2026-05-27T00:01:00Z",
|
||||
"result": {"urls": ["https://krea.cdn/done.png"]},
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.requests.get",
|
||||
return_value=_poll_response(oddball),
|
||||
), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is True
|
||||
|
||||
|
||||
class TestPollRetryPolicy:
|
||||
"""Polling fail-fast on permanent 4xx, retry on transient 5xx/429."""
|
||||
|
||||
def _http_error_response(self, status: int):
|
||||
import requests as req_lib
|
||||
|
||||
resp = req_lib.Response()
|
||||
resp.status_code = status
|
||||
resp._content = b'{"error": "boom"}'
|
||||
resp.headers["Content-Type"] = "application/json"
|
||||
resp.raise_for_status = MagicMock(
|
||||
side_effect=req_lib.HTTPError(response=resp)
|
||||
)
|
||||
return resp
|
||||
|
||||
def test_poll_fails_fast_on_401(self):
|
||||
"""Auth failure mid-poll should not wait the 180s deadline."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
bad_poll = self._http_error_response(401)
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "401" in result["error"]
|
||||
# One call — no retry on permanent auth failure.
|
||||
assert mock_get.call_count == 1
|
||||
|
||||
def test_poll_fails_fast_on_404(self):
|
||||
"""Missing job (404) should surface immediately, not retry for 180s."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
bad_poll = self._http_error_response(404)
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "404" in result["error"]
|
||||
assert mock_get.call_count == 1
|
||||
|
||||
def test_poll_fails_fast_on_403(self):
|
||||
"""Billing/permission failure (403) should not retry."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
bad_poll = self._http_error_response(403)
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch("plugins.image_gen.krea.requests.get", return_value=bad_poll) as mock_get, \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert mock_get.call_count == 1
|
||||
|
||||
def test_poll_retries_on_503_then_succeeds(self):
|
||||
"""Transient 5xx should retry and eventually surface a completion."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
flaky = self._http_error_response(503)
|
||||
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.requests.get",
|
||||
side_effect=[flaky, flaky, good],
|
||||
) as mock_get, \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is True
|
||||
assert mock_get.call_count == 3
|
||||
|
||||
def test_poll_retries_on_429(self):
|
||||
"""Rate-limit (429) is in the retryable set."""
|
||||
from plugins.image_gen.krea import KreaImageGenProvider
|
||||
|
||||
rate_limited = self._http_error_response(429)
|
||||
good = _poll_response(_completed_job("https://krea.cdn/ok.png"))
|
||||
|
||||
with patch("plugins.image_gen.krea.requests.post", return_value=_submit_response()), \
|
||||
patch(
|
||||
"plugins.image_gen.krea.requests.get",
|
||||
side_effect=[rate_limited, good],
|
||||
) as mock_get, \
|
||||
patch(
|
||||
"plugins.image_gen.krea.save_url_image",
|
||||
return_value=Path("/tmp/x.png"),
|
||||
), \
|
||||
patch("plugins.image_gen.krea.time.sleep"):
|
||||
result = KreaImageGenProvider().generate(prompt="test")
|
||||
|
||||
assert result["success"] is True
|
||||
assert mock_get.call_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistration:
|
||||
def test_register(self):
|
||||
from plugins.image_gen.krea import KreaImageGenProvider, register
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
register(mock_ctx)
|
||||
mock_ctx.register_image_gen_provider.assert_called_once()
|
||||
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
|
||||
assert isinstance(provider, KreaImageGenProvider)
|
||||
assert provider.name == "krea"
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Tests for the bundled ``openai-codex`` image_gen plugin.
|
||||
|
||||
Mirrors ``test_openai_provider.py`` but targets the standalone
|
||||
Codex/ChatGPT-OAuth-backed provider that uses the Responses
|
||||
``image_generation`` tool path instead of the ``images.generate`` REST
|
||||
endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# The plugin directory uses a hyphen, which is not a valid Python identifier
|
||||
# for the dotted-import form. Load it via importlib so tests don't need to
|
||||
# touch sys.path or rename the directory.
|
||||
codex_plugin = importlib.import_module("plugins.image_gen.openai-codex")
|
||||
|
||||
|
||||
# 1×1 transparent PNG — valid bytes for save_b64_image()
|
||||
_PNG_HEX = (
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
|
||||
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
|
||||
"ae426082"
|
||||
)
|
||||
|
||||
|
||||
def _b64_png() -> str:
|
||||
import base64
|
||||
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _tmp_hermes_home(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
yield tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider(monkeypatch):
|
||||
# Codex plugin is API-key-independent; clear it to make the test honest.
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
return codex_plugin.OpenAICodexImageGenProvider()
|
||||
|
||||
|
||||
# ── Metadata ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMetadata:
|
||||
def test_name(self, provider):
|
||||
assert provider.name == "openai-codex"
|
||||
|
||||
def test_display_name(self, provider):
|
||||
assert provider.display_name == "OpenAI (Codex auth)"
|
||||
|
||||
def test_default_model(self, provider):
|
||||
assert provider.default_model() == "gpt-image-2-medium"
|
||||
|
||||
def test_list_models_three_tiers(self, provider):
|
||||
ids = [m["id"] for m in provider.list_models()]
|
||||
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
|
||||
|
||||
def test_setup_schema_has_no_required_env_vars(self, provider):
|
||||
schema = provider.get_setup_schema()
|
||||
assert schema["env_vars"] == []
|
||||
assert schema["badge"] == "free"
|
||||
|
||||
|
||||
# ── Availability ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestAvailability:
|
||||
def test_unavailable_without_codex_token(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
|
||||
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
|
||||
|
||||
def test_available_with_codex_token(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
|
||||
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is True
|
||||
|
||||
def test_openai_api_key_alone_is_not_enough(self, monkeypatch):
|
||||
# Codex plugin is intentionally orthogonal to the API-key plugin —
|
||||
# the API key alone must NOT make it appear available.
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
|
||||
assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False
|
||||
|
||||
|
||||
# ── Generate ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGenerate:
|
||||
def test_returns_auth_error_without_codex_token(self, provider, monkeypatch):
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None)
|
||||
result = provider.generate("a cat")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
def test_returns_invalid_argument_for_empty_prompt(self, provider, monkeypatch):
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
|
||||
result = provider.generate(" ")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "invalid_argument"
|
||||
|
||||
def test_generate_uses_codex_stream_path(self, provider, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
|
||||
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: _b64_png())
|
||||
|
||||
result = provider.generate("a cat", aspect_ratio="landscape")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["model"] == "gpt-image-2-medium"
|
||||
assert result["provider"] == "openai-codex"
|
||||
assert result["quality"] == "medium"
|
||||
|
||||
saved = Path(result["image"])
|
||||
assert saved.exists()
|
||||
assert saved.parent == tmp_path / "cache" / "images"
|
||||
# Filename prefix differs from the API-key plugin so cache audits can
|
||||
# tell the two backends apart.
|
||||
assert saved.name.startswith("openai_codex_")
|
||||
|
||||
def test_codex_stream_request_shape(self, provider, monkeypatch):
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
|
||||
|
||||
captured = {}
|
||||
|
||||
def _collect(token, *, prompt, size, quality):
|
||||
captured.update(codex_plugin._build_responses_payload(
|
||||
prompt=prompt,
|
||||
size=size,
|
||||
quality=quality,
|
||||
))
|
||||
return _b64_png()
|
||||
|
||||
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _collect)
|
||||
|
||||
result = provider.generate("a cat", aspect_ratio="portrait")
|
||||
assert result["success"] is True
|
||||
|
||||
assert captured["model"] == "gpt-5.4"
|
||||
assert captured["store"] is False
|
||||
assert captured["input"][0]["type"] == "message"
|
||||
assert captured["input"][0]["role"] == "user"
|
||||
assert captured["input"][0]["content"][0]["type"] == "input_text"
|
||||
assert captured["tool_choice"]["type"] == "allowed_tools"
|
||||
assert captured["tool_choice"]["mode"] == "required"
|
||||
assert captured["tool_choice"]["tools"] == [{"type": "image_generation"}]
|
||||
|
||||
tool = captured["tools"][0]
|
||||
assert tool["type"] == "image_generation"
|
||||
assert tool["model"] == "gpt-image-2"
|
||||
assert tool["quality"] == "medium"
|
||||
assert tool["size"] == "1024x1536"
|
||||
assert tool["output_format"] == "png"
|
||||
assert tool["background"] == "opaque"
|
||||
assert tool["partial_images"] == 1
|
||||
|
||||
def test_partial_image_event_used_when_done_missing(self):
|
||||
"""If output_item.done is missing, partial_image_b64 is accepted."""
|
||||
payload = {
|
||||
"type": "response.image_generation_call.partial_image",
|
||||
"partial_image_b64": _b64_png(),
|
||||
}
|
||||
assert codex_plugin._extract_image_b64(payload) == _b64_png()
|
||||
|
||||
def test_sse_parser_handles_event_and_data_lines(self):
|
||||
class _Response:
|
||||
def iter_lines(self):
|
||||
return iter([
|
||||
"event: response.output_item.done",
|
||||
'data: {"item": {"type": "image_generation_call", "result": "abc"}}',
|
||||
"",
|
||||
])
|
||||
|
||||
events = list(codex_plugin._iter_sse_json(_Response()))
|
||||
assert events == [{
|
||||
"type": "response.output_item.done",
|
||||
"item": {"type": "image_generation_call", "result": "abc"},
|
||||
}]
|
||||
|
||||
def test_final_response_sweep_recovers_image(self):
|
||||
"""Completed response output is found by recursive payload scanning."""
|
||||
payload = {
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"output": [{
|
||||
"type": "image_generation_call",
|
||||
"status": "completed",
|
||||
"id": "ig_final",
|
||||
"result": _b64_png(),
|
||||
}],
|
||||
},
|
||||
}
|
||||
assert codex_plugin._extract_image_b64(payload) == _b64_png()
|
||||
|
||||
def test_empty_response_returns_error(self, provider, monkeypatch):
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
|
||||
monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: None)
|
||||
|
||||
result = provider.generate("a cat")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "empty_response"
|
||||
|
||||
def test_stream_exception_returns_api_error(self, provider, monkeypatch):
|
||||
monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
|
||||
|
||||
def _boom(*args, **kwargs):
|
||||
raise RuntimeError("cloudflare 403")
|
||||
|
||||
monkeypatch.setattr(codex_plugin, "_collect_image_b64", _boom)
|
||||
|
||||
result = provider.generate("a cat")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "cloudflare 403" in result["error"]
|
||||
|
||||
|
||||
# ── Plugin entry point ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRegistration:
|
||||
def test_register_calls_register_image_gen_provider(self):
|
||||
registered = []
|
||||
|
||||
class _Ctx:
|
||||
def register_image_gen_provider(self, prov):
|
||||
registered.append(prov)
|
||||
|
||||
codex_plugin.register(_Ctx())
|
||||
assert len(registered) == 1
|
||||
assert registered[0].name == "openai-codex"
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Tests for the bundled OpenAI image_gen plugin (gpt-image-2, three tiers)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import plugins.image_gen.openai as openai_plugin
|
||||
|
||||
|
||||
# 1×1 transparent PNG — valid bytes for save_b64_image()
|
||||
_PNG_HEX = (
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
|
||||
"890000000d49444154789c6300010000000500010d0a2db40000000049454e44"
|
||||
"ae426082"
|
||||
)
|
||||
|
||||
|
||||
def _b64_png() -> str:
|
||||
import base64
|
||||
return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode()
|
||||
|
||||
|
||||
def _fake_response(*, b64=None, url=None, revised_prompt=None):
|
||||
item = SimpleNamespace(b64_json=b64, url=url, revised_prompt=revised_prompt)
|
||||
return SimpleNamespace(data=[item])
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _tmp_hermes_home(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
yield tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider(monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
|
||||
return openai_plugin.OpenAIImageGenProvider()
|
||||
|
||||
|
||||
def _patched_openai(fake_client: MagicMock):
|
||||
fake_openai = MagicMock()
|
||||
fake_openai.OpenAI.return_value = fake_client
|
||||
return patch.dict("sys.modules", {"openai": fake_openai})
|
||||
|
||||
|
||||
# ── Metadata ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMetadata:
|
||||
def test_name(self, provider):
|
||||
assert provider.name == "openai"
|
||||
|
||||
def test_default_model(self, provider):
|
||||
assert provider.default_model() == "gpt-image-2-medium"
|
||||
|
||||
def test_list_models_three_tiers(self, provider):
|
||||
ids = [m["id"] for m in provider.list_models()]
|
||||
assert ids == ["gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"]
|
||||
|
||||
def test_catalog_entries_have_display_speed_strengths(self, provider):
|
||||
for entry in provider.list_models():
|
||||
assert entry["display"].startswith("GPT Image 2")
|
||||
assert entry["speed"]
|
||||
assert entry["strengths"]
|
||||
|
||||
|
||||
# ── Availability ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestAvailability:
|
||||
def test_no_api_key_unavailable(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
assert openai_plugin.OpenAIImageGenProvider().is_available() is False
|
||||
|
||||
def test_api_key_set_available(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "test")
|
||||
assert openai_plugin.OpenAIImageGenProvider().is_available() is True
|
||||
|
||||
|
||||
# ── Model resolution ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestModelResolution:
|
||||
def test_default_is_medium(self):
|
||||
model_id, meta = openai_plugin._resolve_model()
|
||||
assert model_id == "gpt-image-2-medium"
|
||||
assert meta["quality"] == "medium"
|
||||
|
||||
def test_env_var_override(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "gpt-image-2-high")
|
||||
model_id, meta = openai_plugin._resolve_model()
|
||||
assert model_id == "gpt-image-2-high"
|
||||
assert meta["quality"] == "high"
|
||||
|
||||
def test_env_var_unknown_falls_back(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_IMAGE_MODEL", "bogus-tier")
|
||||
model_id, _ = openai_plugin._resolve_model()
|
||||
assert model_id == openai_plugin.DEFAULT_MODEL
|
||||
|
||||
def test_config_openai_model(self, tmp_path):
|
||||
import yaml
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"image_gen": {"openai": {"model": "gpt-image-2-low"}}})
|
||||
)
|
||||
model_id, meta = openai_plugin._resolve_model()
|
||||
assert model_id == "gpt-image-2-low"
|
||||
assert meta["quality"] == "low"
|
||||
|
||||
def test_config_top_level_model(self, tmp_path):
|
||||
"""``image_gen.model: gpt-image-2-high`` also works (top-level)."""
|
||||
import yaml
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"image_gen": {"model": "gpt-image-2-high"}})
|
||||
)
|
||||
model_id, meta = openai_plugin._resolve_model()
|
||||
assert model_id == "gpt-image-2-high"
|
||||
assert meta["quality"] == "high"
|
||||
|
||||
|
||||
# ── Generate ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGenerate:
|
||||
def test_empty_prompt_rejected(self, provider):
|
||||
result = provider.generate("", aspect_ratio="square")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "invalid_argument"
|
||||
|
||||
def test_missing_api_key(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
result = openai_plugin.OpenAIImageGenProvider().generate("a cat")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
def test_b64_saves_to_cache(self, provider, tmp_path):
|
||||
png_bytes = bytes.fromhex(_PNG_HEX)
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
|
||||
|
||||
with _patched_openai(fake_client):
|
||||
result = provider.generate("a cat", aspect_ratio="landscape")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["model"] == "gpt-image-2-medium"
|
||||
assert result["aspect_ratio"] == "landscape"
|
||||
assert result["provider"] == "openai"
|
||||
assert result["quality"] == "medium"
|
||||
|
||||
saved = Path(result["image"])
|
||||
assert saved.exists()
|
||||
assert saved.parent == tmp_path / "cache" / "images"
|
||||
assert saved.read_bytes() == png_bytes
|
||||
|
||||
call_kwargs = fake_client.images.generate.call_args.kwargs
|
||||
# All tiers hit the single underlying API model.
|
||||
assert call_kwargs["model"] == "gpt-image-2"
|
||||
assert call_kwargs["quality"] == "medium"
|
||||
assert call_kwargs["size"] == "1536x1024"
|
||||
# gpt-image-2 rejects response_format — we must NOT send it.
|
||||
assert "response_format" not in call_kwargs
|
||||
|
||||
@pytest.mark.parametrize("tier,expected_quality", [
|
||||
("gpt-image-2-low", "low"),
|
||||
("gpt-image-2-medium", "medium"),
|
||||
("gpt-image-2-high", "high"),
|
||||
])
|
||||
def test_tier_maps_to_quality(self, provider, monkeypatch, tier, expected_quality):
|
||||
monkeypatch.setenv("OPENAI_IMAGE_MODEL", tier)
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
|
||||
|
||||
with _patched_openai(fake_client):
|
||||
result = provider.generate("a cat")
|
||||
|
||||
assert result["model"] == tier
|
||||
assert result["quality"] == expected_quality
|
||||
assert fake_client.images.generate.call_args.kwargs["quality"] == expected_quality
|
||||
# Always the same underlying API model regardless of tier.
|
||||
assert fake_client.images.generate.call_args.kwargs["model"] == "gpt-image-2"
|
||||
|
||||
@pytest.mark.parametrize("aspect,expected_size", [
|
||||
("landscape", "1536x1024"),
|
||||
("square", "1024x1024"),
|
||||
("portrait", "1024x1536"),
|
||||
])
|
||||
def test_aspect_ratio_mapping(self, provider, aspect, expected_size):
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = _fake_response(b64=_b64_png())
|
||||
|
||||
with _patched_openai(fake_client):
|
||||
provider.generate("a cat", aspect_ratio=aspect)
|
||||
|
||||
assert fake_client.images.generate.call_args.kwargs["size"] == expected_size
|
||||
|
||||
def test_revised_prompt_passed_through(self, provider):
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = _fake_response(
|
||||
b64=_b64_png(), revised_prompt="A photo of a cat",
|
||||
)
|
||||
|
||||
with _patched_openai(fake_client):
|
||||
result = provider.generate("a cat")
|
||||
|
||||
assert result["revised_prompt"] == "A photo of a cat"
|
||||
|
||||
def test_api_error_returns_error_response(self, provider):
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.side_effect = RuntimeError("boom")
|
||||
|
||||
with _patched_openai(fake_client):
|
||||
result = provider.generate("a cat")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "boom" in result["error"]
|
||||
|
||||
def test_empty_response_data(self, provider):
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = SimpleNamespace(data=[])
|
||||
|
||||
with _patched_openai(fake_client):
|
||||
result = provider.generate("a cat")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "empty_response"
|
||||
|
||||
def test_url_response_is_cached_locally(self, provider):
|
||||
"""OpenAI URL response (if API ever returns one) is cached locally.
|
||||
|
||||
Pre-fix this asserted the bare URL passed through; symmetric to the
|
||||
xAI #26942 fix. Even though gpt-image-2 returns b64 today, every
|
||||
``image_gen`` provider must guarantee the gateway gets a stable
|
||||
file path so ephemeral signed URLs can't expire mid-flight.
|
||||
"""
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = _fake_response(
|
||||
b64=None, url="https://example.com/img.png",
|
||||
)
|
||||
|
||||
with _patched_openai(fake_client), patch(
|
||||
"plugins.image_gen.openai.save_url_image",
|
||||
return_value=Path("/tmp/openai_gpt-image-2_20260524_000000_deadbeef.png"),
|
||||
) as mock_save_url:
|
||||
result = provider.generate("a cat")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image"].startswith("/")
|
||||
assert "example.com" not in result["image"]
|
||||
mock_save_url.assert_called_once()
|
||||
|
||||
def test_url_response_falls_back_to_bare_url_when_download_fails(self, provider):
|
||||
"""Cache failure must not turn into a tool error — symmetric with xAI."""
|
||||
import requests as req_lib
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.images.generate.return_value = _fake_response(
|
||||
b64=None, url="https://example.com/img.png",
|
||||
)
|
||||
|
||||
with _patched_openai(fake_client), patch(
|
||||
"plugins.image_gen.openai.save_url_image",
|
||||
side_effect=req_lib.HTTPError("404 from CDN"),
|
||||
):
|
||||
result = provider.generate("a cat")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image"] == "https://example.com/img.png"
|
||||
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for xAI image generation provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fake_api_key(monkeypatch):
|
||||
"""Ensure XAI_API_KEY is set for all tests."""
|
||||
monkeypatch.setenv("XAI_API_KEY", "test-key-12345")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider class tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestXAIImageGenProvider:
|
||||
def test_name(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
assert provider.name == "xai"
|
||||
|
||||
def test_display_name(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
assert provider.display_name == "xAI (Grok)"
|
||||
|
||||
def test_is_available_with_key(self, monkeypatch):
|
||||
monkeypatch.setenv("XAI_API_KEY", "sk-xxx")
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
assert provider.is_available() is True
|
||||
|
||||
def test_is_available_without_key(self, monkeypatch):
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
assert provider.is_available() is False
|
||||
|
||||
def test_list_models(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
models = provider.list_models()
|
||||
assert len(models) >= 1
|
||||
assert models[0]["id"] == "grok-imagine-image"
|
||||
|
||||
def test_default_model(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
assert provider.default_model() == "grok-imagine-image"
|
||||
|
||||
def test_get_setup_schema(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
schema = provider.get_setup_schema()
|
||||
assert schema["name"] == "xAI Grok Imagine (image)"
|
||||
assert schema["badge"] == "paid"
|
||||
# Auth resolution is delegated to the shared "xai_grok" post_setup
|
||||
# hook so the picker doesn't blindly prompt for XAI_API_KEY when the
|
||||
# user is already signed in via xAI Grok OAuth.
|
||||
assert schema["env_vars"] == []
|
||||
assert schema["post_setup"] == "xai_grok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfig:
|
||||
def test_default_model(self):
|
||||
from plugins.image_gen.xai import _resolve_model
|
||||
|
||||
model_id, meta = _resolve_model()
|
||||
assert model_id == "grok-imagine-image"
|
||||
|
||||
def test_default_resolution(self):
|
||||
from plugins.image_gen.xai import _resolve_resolution
|
||||
|
||||
assert _resolve_resolution() == "1k"
|
||||
|
||||
def test_custom_model(self, monkeypatch):
|
||||
monkeypatch.setenv("XAI_IMAGE_MODEL", "grok-imagine-image")
|
||||
from plugins.image_gen.xai import _resolve_model
|
||||
|
||||
model_id, _ = _resolve_model()
|
||||
assert model_id == "grok-imagine-image"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generate tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerate:
|
||||
def test_missing_api_key(self, monkeypatch):
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="test")
|
||||
assert result["success"] is False
|
||||
assert "XAI_API_KEY" in result["error"]
|
||||
|
||||
def test_successful_generation(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json.return_value = {
|
||||
"data": [{"b64_json": "dGVzdC1pbWFnZS1kYXRh"}], # base64 "test-image-data"
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
|
||||
with patch("plugins.image_gen.xai.save_b64_image", return_value="/tmp/test.png"):
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="A cat playing piano")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image"] == "/tmp/test.png"
|
||||
assert result["provider"] == "xai"
|
||||
assert result["model"] == "grok-imagine-image"
|
||||
|
||||
def test_successful_url_response(self):
|
||||
"""xAI URL response is cached locally — #26942 contract.
|
||||
|
||||
Pre-fix this asserted ``result["image"] == "<the bare URL>"``, which
|
||||
was exactly the bug: xAI's ``imgen.x.ai/xai-tmp-*`` URLs expire fast
|
||||
and the gateway 404'd by ``send_photo`` time. Post-fix the URL
|
||||
bytes are downloaded at tool-completion and the result carries an
|
||||
absolute filesystem path the gateway can upload from.
|
||||
"""
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json.return_value = {
|
||||
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"}],
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
|
||||
patch(
|
||||
"plugins.image_gen.xai.save_url_image",
|
||||
return_value=Path("/tmp/xai_grok-imagine-image_20260524_000000_deadbeef.jpg"),
|
||||
) as mock_save_url:
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="A cat playing piano")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["image"].startswith("/"), (
|
||||
f"URL response must be cached to an absolute path, got {result['image']!r}"
|
||||
)
|
||||
assert "imgen.x.ai" not in result["image"], (
|
||||
"ephemeral xAI URL must not leak into result.image — caller will 404"
|
||||
)
|
||||
# The downloader should have been called exactly once with the URL
|
||||
# and an xai-prefixed cache filename.
|
||||
mock_save_url.assert_called_once()
|
||||
call_args, call_kwargs = mock_save_url.call_args
|
||||
assert call_args[0] == "https://imgen.x.ai/xai-tmp-imgen-test.jpeg"
|
||||
assert call_kwargs.get("prefix", "").startswith("xai_")
|
||||
|
||||
def test_url_response_falls_back_to_bare_url_when_download_fails(self):
|
||||
"""If caching the URL fails (network blip, 404 in-flight), the
|
||||
provider must NOT hard-error — fall through to returning the bare
|
||||
URL so the agent surface at least sees *something*. The gateway's
|
||||
existing URL-send fallback then has a chance to succeed; if it
|
||||
too 404s, the user gets the original (now legible) error rather
|
||||
than an opaque "image generation failed" tool result.
|
||||
"""
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json.return_value = {
|
||||
"data": [{"url": "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"}],
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp), \
|
||||
patch(
|
||||
"plugins.image_gen.xai.save_url_image",
|
||||
side_effect=req_lib.HTTPError("404 from CDN"),
|
||||
):
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="A cat playing piano")
|
||||
|
||||
assert result["success"] is True, (
|
||||
"Cache failure must not turn into a tool error — gateway gets a chance to retry"
|
||||
)
|
||||
assert result["image"] == "https://imgen.x.ai/xai-tmp-imgen-already-404.jpeg"
|
||||
|
||||
def test_api_error(self):
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 401
|
||||
mock_resp.text = "Unauthorized"
|
||||
mock_resp.json.return_value = {"error": {"message": "Invalid API key"}}
|
||||
mock_resp.raise_for_status.side_effect = req_lib.HTTPError(response=mock_resp)
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
|
||||
def test_api_error_preserves_real_response_status(self):
|
||||
import requests as req_lib
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
response = req_lib.Response()
|
||||
response.status_code = 401
|
||||
response._content = json.dumps({"error": {"message": "Invalid API key"}}).encode()
|
||||
response.headers["Content-Type"] = "application/json"
|
||||
|
||||
response.raise_for_status = MagicMock(
|
||||
side_effect=req_lib.HTTPError(response=response)
|
||||
)
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=response):
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "api_error"
|
||||
assert "xAI image generation failed (401): Invalid API key" in result["error"]
|
||||
|
||||
def test_timeout(self):
|
||||
import requests as req_lib
|
||||
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", side_effect=req_lib.Timeout()):
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "timeout"
|
||||
|
||||
def test_empty_response(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json.return_value = {"data": []}
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp):
|
||||
provider = XAIImageGenProvider()
|
||||
result = provider.generate(prompt="test")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "empty_response"
|
||||
|
||||
def test_auth_header(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json.return_value = {
|
||||
"data": [{"url": "https://xai.image/test.png"}],
|
||||
}
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
|
||||
provider = XAIImageGenProvider()
|
||||
provider.generate(prompt="test")
|
||||
|
||||
call_args = mock_post.call_args
|
||||
headers = call_args.kwargs.get("headers") or call_args[1].get("headers")
|
||||
assert "Bearer test-key-12345" in headers["Authorization"]
|
||||
assert "Hermes-Agent" in headers["User-Agent"]
|
||||
|
||||
def test_payload_resolution_is_literal_1k_or_2k(self):
|
||||
"""Regression: xAI API rejects numeric resolutions ("1024"/"2048") with 422.
|
||||
|
||||
The endpoint expects the literal strings "1k" or "2k". Ensure the wire
|
||||
payload carries that literal — not a numeric mapping. See PR #18678.
|
||||
"""
|
||||
from plugins.image_gen.xai import XAIImageGenProvider
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json.return_value = {"data": [{"url": "https://xai.image/test.png"}]}
|
||||
|
||||
with patch("plugins.image_gen.xai.requests.post", return_value=mock_resp) as mock_post:
|
||||
provider = XAIImageGenProvider()
|
||||
provider.generate(prompt="test")
|
||||
|
||||
payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json")
|
||||
assert payload["resolution"] in {"1k", "2k"}, (
|
||||
f"resolution must be the literal '1k' or '2k', got {payload['resolution']!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistration:
|
||||
def test_register(self):
|
||||
from plugins.image_gen.xai import XAIImageGenProvider, register
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
register(mock_ctx)
|
||||
mock_ctx.register_image_gen_provider.assert_called_once()
|
||||
provider = mock_ctx.register_image_gen_provider.call_args[0][0]
|
||||
assert isinstance(provider, XAIImageGenProvider)
|
||||
assert provider.name == "xai"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,241 @@
|
||||
"""Tests for Mem0 API v2 compatibility — filters param and dict response unwrapping.
|
||||
|
||||
Salvaged from PRs #5301 (qaqcvc) and #5117 (vvvanguards).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.memory.mem0 import Mem0MemoryProvider
|
||||
|
||||
|
||||
class FakeClientV2:
|
||||
"""Fake Mem0 client that returns v2-style dict responses and captures call kwargs."""
|
||||
|
||||
def __init__(self, search_results=None, all_results=None):
|
||||
self._search_results = search_results or {"results": []}
|
||||
self._all_results = all_results or {"results": []}
|
||||
self.captured_search = {}
|
||||
self.captured_get_all = {}
|
||||
self.captured_add = []
|
||||
|
||||
def search(self, **kwargs):
|
||||
self.captured_search = kwargs
|
||||
return self._search_results
|
||||
|
||||
def get_all(self, **kwargs):
|
||||
self.captured_get_all = kwargs
|
||||
return self._all_results
|
||||
|
||||
def add(self, messages, **kwargs):
|
||||
self.captured_add.append({"messages": messages, **kwargs})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Filter migration: bare user_id= -> filters={}
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMem0FiltersV2:
|
||||
"""All API calls must use filters={} instead of bare user_id= kwargs."""
|
||||
|
||||
def _make_provider(self, monkeypatch, client):
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.initialize("test-session")
|
||||
provider._user_id = "u123"
|
||||
provider._agent_id = "hermes"
|
||||
monkeypatch.setattr(provider, "_get_client", lambda: client)
|
||||
return provider
|
||||
|
||||
def test_search_uses_filters(self, monkeypatch):
|
||||
client = FakeClientV2()
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
provider.handle_tool_call("mem0_search", {"query": "hello", "top_k": 3, "rerank": False})
|
||||
|
||||
assert client.captured_search["query"] == "hello"
|
||||
assert client.captured_search["top_k"] == 3
|
||||
assert client.captured_search["rerank"] is False
|
||||
assert client.captured_search["filters"] == {"user_id": "u123"}
|
||||
# Must NOT have bare user_id kwarg
|
||||
assert "user_id" not in {k for k in client.captured_search if k != "filters"}
|
||||
|
||||
def test_profile_uses_filters(self, monkeypatch):
|
||||
client = FakeClientV2()
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
provider.handle_tool_call("mem0_profile", {})
|
||||
|
||||
assert client.captured_get_all["filters"] == {"user_id": "u123"}
|
||||
assert "user_id" not in {k for k in client.captured_get_all if k != "filters"}
|
||||
|
||||
def test_prefetch_uses_filters(self, monkeypatch):
|
||||
client = FakeClientV2()
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
provider.queue_prefetch("hello")
|
||||
provider._prefetch_thread.join(timeout=2)
|
||||
|
||||
assert client.captured_search["query"] == "hello"
|
||||
assert client.captured_search["filters"] == {"user_id": "u123"}
|
||||
assert "user_id" not in {k for k in client.captured_search if k != "filters"}
|
||||
|
||||
def test_sync_turn_uses_write_filters(self, monkeypatch):
|
||||
client = FakeClientV2()
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
provider.sync_turn("user said this", "assistant replied", session_id="s1")
|
||||
provider._sync_thread.join(timeout=2)
|
||||
|
||||
assert len(client.captured_add) == 1
|
||||
call = client.captured_add[0]
|
||||
assert call["user_id"] == "u123"
|
||||
assert call["agent_id"] == "hermes"
|
||||
|
||||
def test_conclude_uses_write_filters(self, monkeypatch):
|
||||
client = FakeClientV2()
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
provider.handle_tool_call("mem0_conclude", {"conclusion": "user likes dark mode"})
|
||||
|
||||
assert len(client.captured_add) == 1
|
||||
call = client.captured_add[0]
|
||||
assert call["user_id"] == "u123"
|
||||
assert call["agent_id"] == "hermes"
|
||||
assert call["infer"] is False
|
||||
|
||||
def test_read_filters_no_agent_id(self):
|
||||
"""Read filters should use user_id only — cross-session recall across agents."""
|
||||
provider = Mem0MemoryProvider()
|
||||
provider._user_id = "u123"
|
||||
provider._agent_id = "hermes"
|
||||
assert provider._read_filters() == {"user_id": "u123"}
|
||||
|
||||
def test_write_filters_include_agent_id(self):
|
||||
"""Write filters should include agent_id for attribution."""
|
||||
provider = Mem0MemoryProvider()
|
||||
provider._user_id = "u123"
|
||||
provider._agent_id = "hermes"
|
||||
assert provider._write_filters() == {"user_id": "u123", "agent_id": "hermes"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dict response unwrapping (API v2 wraps in {"results": [...]})
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMem0ResponseUnwrapping:
|
||||
"""API v2 returns {"results": [...]} dicts; we must extract the list."""
|
||||
|
||||
def _make_provider(self, monkeypatch, client):
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.initialize("test-session")
|
||||
monkeypatch.setattr(provider, "_get_client", lambda: client)
|
||||
return provider
|
||||
|
||||
def test_profile_dict_response(self, monkeypatch):
|
||||
client = FakeClientV2(all_results={"results": [{"memory": "alpha"}, {"memory": "beta"}]})
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
result = json.loads(provider.handle_tool_call("mem0_profile", {}))
|
||||
|
||||
assert result["count"] == 2
|
||||
assert "alpha" in result["result"]
|
||||
assert "beta" in result["result"]
|
||||
|
||||
def test_profile_list_response_backward_compat(self, monkeypatch):
|
||||
"""Old API returned bare lists — still works."""
|
||||
client = FakeClientV2(all_results=[{"memory": "gamma"}])
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
result = json.loads(provider.handle_tool_call("mem0_profile", {}))
|
||||
assert result["count"] == 1
|
||||
assert "gamma" in result["result"]
|
||||
|
||||
def test_search_dict_response(self, monkeypatch):
|
||||
client = FakeClientV2(search_results={
|
||||
"results": [{"memory": "foo", "score": 0.9}, {"memory": "bar", "score": 0.7}]
|
||||
})
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
result = json.loads(provider.handle_tool_call(
|
||||
"mem0_search", {"query": "test", "top_k": 5}
|
||||
))
|
||||
|
||||
assert result["count"] == 2
|
||||
assert result["results"][0]["memory"] == "foo"
|
||||
|
||||
def test_search_list_response_backward_compat(self, monkeypatch):
|
||||
"""Old API returned bare lists — still works."""
|
||||
client = FakeClientV2(search_results=[{"memory": "baz", "score": 0.8}])
|
||||
provider = self._make_provider(monkeypatch, client)
|
||||
|
||||
result = json.loads(provider.handle_tool_call(
|
||||
"mem0_search", {"query": "test"}
|
||||
))
|
||||
assert result["count"] == 1
|
||||
|
||||
def test_unwrap_results_edge_cases(self):
|
||||
"""_unwrap_results handles all shapes gracefully."""
|
||||
assert Mem0MemoryProvider._unwrap_results({"results": [1, 2]}) == [1, 2]
|
||||
assert Mem0MemoryProvider._unwrap_results([3, 4]) == [3, 4]
|
||||
assert Mem0MemoryProvider._unwrap_results({}) == []
|
||||
assert Mem0MemoryProvider._unwrap_results(None) == []
|
||||
assert Mem0MemoryProvider._unwrap_results("unexpected") == []
|
||||
|
||||
def test_prefetch_dict_response(self, monkeypatch):
|
||||
client = FakeClientV2(search_results={
|
||||
"results": [{"memory": "user prefers dark mode"}]
|
||||
})
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.initialize("test-session")
|
||||
monkeypatch.setattr(provider, "_get_client", lambda: client)
|
||||
|
||||
provider.queue_prefetch("preferences")
|
||||
provider._prefetch_thread.join(timeout=2)
|
||||
result = provider.prefetch("preferences")
|
||||
|
||||
assert "dark mode" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default preservation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode bits not enforced on Windows")
|
||||
def test_save_config_sets_owner_only_permissions(tmp_path):
|
||||
"""mem0.json must be written with 0o600 so API key is not world-readable."""
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.save_config({"api_key": "m0-test-key"}, str(tmp_path))
|
||||
config_file = tmp_path / "mem0.json"
|
||||
assert config_file.exists()
|
||||
mode = stat.S_IMODE(config_file.stat().st_mode)
|
||||
assert mode == 0o600, f"Expected 0o600 (owner-only), got {oct(mode)}"
|
||||
|
||||
|
||||
class TestMem0Defaults:
|
||||
"""Ensure we don't break existing users' defaults."""
|
||||
|
||||
def test_default_user_id_hermes_user(self, monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("MEM0_API_KEY", "test-key")
|
||||
monkeypatch.delenv("MEM0_USER_ID", raising=False)
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.initialize("test")
|
||||
|
||||
assert provider._user_id == "hermes-user"
|
||||
|
||||
def test_default_agent_id_hermes(self, monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("MEM0_API_KEY", "test-key")
|
||||
monkeypatch.delenv("MEM0_AGENT_ID", raising=False)
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
provider = Mem0MemoryProvider()
|
||||
provider.initialize("test")
|
||||
|
||||
assert provider._agent_id == "hermes"
|
||||
@@ -0,0 +1,422 @@
|
||||
import json
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.memory.openviking import OpenVikingMemoryProvider, _VikingClient
|
||||
|
||||
|
||||
def test_tool_search_sorts_by_raw_score_across_buckets():
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
provider._client.post.return_value = {
|
||||
"result": {
|
||||
"memories": [
|
||||
{"uri": "viking://memories/1", "score": 0.9003, "abstract": "memory result"},
|
||||
],
|
||||
"resources": [
|
||||
{"uri": "viking://resources/1", "score": 0.9004, "abstract": "resource result"},
|
||||
],
|
||||
"skills": [
|
||||
{"uri": "viking://skills/1", "score": 0.8999, "abstract": "skill result"},
|
||||
],
|
||||
"total": 3,
|
||||
}
|
||||
}
|
||||
|
||||
result = json.loads(provider._tool_search({"query": "ranking"}))
|
||||
|
||||
assert [entry["uri"] for entry in result["results"]] == [
|
||||
"viking://resources/1",
|
||||
"viking://memories/1",
|
||||
"viking://skills/1",
|
||||
]
|
||||
assert [entry["score"] for entry in result["results"]] == [0.9, 0.9, 0.9]
|
||||
assert result["total"] == 3
|
||||
|
||||
|
||||
def test_tool_search_sorts_missing_raw_score_after_negative_scores():
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
provider._client.post.return_value = {
|
||||
"result": {
|
||||
"memories": [
|
||||
{"uri": "viking://memories/missing", "abstract": "missing score"},
|
||||
],
|
||||
"resources": [
|
||||
{"uri": "viking://resources/negative", "score": -0.25, "abstract": "negative score"},
|
||||
],
|
||||
"skills": [
|
||||
{"uri": "viking://skills/positive", "score": 0.1, "abstract": "positive score"},
|
||||
],
|
||||
"total": 3,
|
||||
}
|
||||
}
|
||||
|
||||
result = json.loads(provider._tool_search({"query": "ranking"}))
|
||||
|
||||
assert [entry["uri"] for entry in result["results"]] == [
|
||||
"viking://skills/positive",
|
||||
"viking://memories/missing",
|
||||
"viking://resources/negative",
|
||||
]
|
||||
assert [entry["score"] for entry in result["results"]] == [0.1, 0.0, -0.25]
|
||||
assert result["total"] == 3
|
||||
|
||||
|
||||
def test_tool_add_resource_uploads_existing_local_file(tmp_path):
|
||||
sample = tmp_path / "sample.md"
|
||||
sample.write_text("# Local resource\n", encoding="utf-8")
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
provider._client.upload_temp_file.return_value = "upload_sample.md"
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/sample"},
|
||||
}
|
||||
|
||||
result = json.loads(provider._tool_add_resource({
|
||||
"url": str(sample),
|
||||
"reason": "local test",
|
||||
"wait": True,
|
||||
}))
|
||||
|
||||
provider._client.upload_temp_file.assert_called_once_with(sample)
|
||||
provider._client.post.assert_called_once_with("/api/v1/resources", {
|
||||
"reason": "local test",
|
||||
"wait": True,
|
||||
"source_name": "sample.md",
|
||||
"temp_file_id": "upload_sample.md",
|
||||
})
|
||||
assert result["status"] == "added"
|
||||
assert result["root_uri"] == "viking://resources/sample"
|
||||
|
||||
|
||||
def test_tool_add_resource_uploads_file_uri(tmp_path):
|
||||
sample = tmp_path / "sample.md"
|
||||
sample.write_text("# Local resource\n", encoding="utf-8")
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
provider._client.upload_temp_file.return_value = "upload_sample.md"
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/sample"},
|
||||
}
|
||||
|
||||
result = json.loads(provider._tool_add_resource({
|
||||
"url": sample.as_uri(),
|
||||
"reason": "file uri test",
|
||||
}))
|
||||
|
||||
provider._client.upload_temp_file.assert_called_once_with(sample)
|
||||
provider._client.post.assert_called_once_with("/api/v1/resources", {
|
||||
"reason": "file uri test",
|
||||
"source_name": "sample.md",
|
||||
"temp_file_id": "upload_sample.md",
|
||||
})
|
||||
assert result["status"] == "added"
|
||||
assert result["root_uri"] == "viking://resources/sample"
|
||||
|
||||
|
||||
def test_tool_add_resource_uploads_existing_local_directory_and_cleans_zip(tmp_path):
|
||||
docs = tmp_path / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
|
||||
nested = docs / "nested"
|
||||
nested.mkdir()
|
||||
(nested / "api.md").write_text("# API\n", encoding="utf-8")
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
uploaded_paths = []
|
||||
provider._client.upload_temp_file.side_effect = (
|
||||
lambda path: uploaded_paths.append(path) or "upload_docs.zip"
|
||||
)
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/docs"},
|
||||
}
|
||||
|
||||
result = json.loads(provider._tool_add_resource({
|
||||
"url": str(docs),
|
||||
"reason": "directory test",
|
||||
"wait": True,
|
||||
}))
|
||||
|
||||
assert uploaded_paths
|
||||
assert uploaded_paths[0].suffix == ".zip"
|
||||
assert not uploaded_paths[0].exists()
|
||||
provider._client.post.assert_called_once_with("/api/v1/resources", {
|
||||
"reason": "directory test",
|
||||
"wait": True,
|
||||
"source_name": "docs",
|
||||
"temp_file_id": "upload_docs.zip",
|
||||
})
|
||||
assert result["status"] == "added"
|
||||
assert result["root_uri"] == "viking://resources/docs"
|
||||
|
||||
|
||||
def test_tool_add_resource_directory_zip_skips_symlink_escape(tmp_path):
|
||||
secret = tmp_path / "outside-secret.txt"
|
||||
secret.write_text("do not upload\n", encoding="utf-8")
|
||||
docs = tmp_path / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
|
||||
link = docs / "leak.txt"
|
||||
try:
|
||||
link.symlink_to(secret)
|
||||
except OSError as exc:
|
||||
pytest.skip(f"symlinks unavailable in test environment: {exc}")
|
||||
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
archive_entries = {}
|
||||
|
||||
def inspect_upload(path):
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
archive_entries["names"] = archive.namelist()
|
||||
archive_entries["payloads"] = {
|
||||
name: archive.read(name)
|
||||
for name in archive.namelist()
|
||||
}
|
||||
return "upload_docs.zip"
|
||||
|
||||
provider._client.upload_temp_file.side_effect = inspect_upload
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/docs"},
|
||||
}
|
||||
|
||||
json.loads(provider._tool_add_resource({"url": str(docs)}))
|
||||
|
||||
assert archive_entries["names"] == ["guide.md"]
|
||||
assert b"do not upload" not in b"".join(archive_entries["payloads"].values())
|
||||
|
||||
|
||||
def test_tool_add_resource_cleans_local_directory_zip_when_add_fails(tmp_path):
|
||||
docs = tmp_path / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
uploaded_paths = []
|
||||
provider._client.upload_temp_file.side_effect = (
|
||||
lambda path: uploaded_paths.append(path) or "upload_docs.zip"
|
||||
)
|
||||
provider._client.post.side_effect = RuntimeError("add failed")
|
||||
|
||||
with pytest.raises(RuntimeError, match="add failed"):
|
||||
provider._tool_add_resource({"url": str(docs)})
|
||||
|
||||
assert uploaded_paths
|
||||
assert not uploaded_paths[0].exists()
|
||||
|
||||
|
||||
def test_tool_add_resource_cleans_local_directory_zip_when_upload_fails(tmp_path):
|
||||
docs = tmp_path / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
uploaded_paths = []
|
||||
|
||||
def fail_upload(path):
|
||||
uploaded_paths.append(path)
|
||||
raise RuntimeError("upload failed")
|
||||
|
||||
provider._client.upload_temp_file.side_effect = fail_upload
|
||||
|
||||
with pytest.raises(RuntimeError, match="upload failed"):
|
||||
provider._tool_add_resource({"url": str(docs)})
|
||||
|
||||
assert uploaded_paths
|
||||
assert not uploaded_paths[0].exists()
|
||||
provider._client.post.assert_not_called()
|
||||
|
||||
|
||||
def test_tool_add_resource_rejects_missing_local_path(tmp_path):
|
||||
missing = tmp_path / "missing.md"
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
|
||||
result = json.loads(provider._tool_add_resource({"url": str(missing)}))
|
||||
|
||||
assert result["error"] == f"Local resource path does not exist: {missing}"
|
||||
provider._client.upload_temp_file.assert_not_called()
|
||||
provider._client.post.assert_not_called()
|
||||
|
||||
|
||||
def test_tool_add_resource_sends_remote_url_as_path():
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/remote"},
|
||||
}
|
||||
|
||||
provider._tool_add_resource({"url": "https://example.com/doc.md"})
|
||||
|
||||
provider._client.upload_temp_file.assert_not_called()
|
||||
provider._client.post.assert_called_once_with("/api/v1/resources", {
|
||||
"path": "https://example.com/doc.md",
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"git@github.com:org/repo.git",
|
||||
"git@ssh.dev.azure.com:v3/org/project/repo",
|
||||
"ssh://git@github.com/org/repo.git",
|
||||
"git://github.com/org/repo.git",
|
||||
])
|
||||
def test_tool_add_resource_sends_git_remote_sources_as_path(url):
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/repo"},
|
||||
}
|
||||
|
||||
provider._tool_add_resource({"url": url})
|
||||
|
||||
provider._client.upload_temp_file.assert_not_called()
|
||||
provider._client.post.assert_called_once_with("/api/v1/resources", {
|
||||
"path": url,
|
||||
})
|
||||
|
||||
|
||||
def test_viking_client_upload_temp_file_uses_multipart_identity_headers(tmp_path, monkeypatch):
|
||||
sample = tmp_path / "sample.md"
|
||||
sample.write_text("# Local resource\n", encoding="utf-8")
|
||||
client = _VikingClient(
|
||||
"https://example.com",
|
||||
api_key="test-key",
|
||||
account="test-account",
|
||||
user="test-user",
|
||||
agent="test-agent",
|
||||
)
|
||||
captured_kwargs = {}
|
||||
|
||||
def capture_httpx_post(url, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return SimpleNamespace(
|
||||
status_code=200,
|
||||
text="",
|
||||
json=lambda: {"status": "ok", "result": {"temp_file_id": "upload_sample.md"}},
|
||||
raise_for_status=lambda: None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(client._httpx, "post", capture_httpx_post)
|
||||
|
||||
assert client.upload_temp_file(sample) == "upload_sample.md"
|
||||
|
||||
assert "files" in captured_kwargs
|
||||
assert "json" not in captured_kwargs
|
||||
headers = captured_kwargs["headers"]
|
||||
assert headers["X-OpenViking-Account"] == "test-account"
|
||||
assert headers["X-OpenViking-User"] == "test-user"
|
||||
assert headers["X-OpenViking-Agent"] == "test-agent"
|
||||
assert headers["X-API-Key"] == "test-key"
|
||||
assert "Content-Type" not in headers
|
||||
|
||||
|
||||
def test_viking_client_raises_structured_server_error():
|
||||
client = _VikingClient.__new__(_VikingClient)
|
||||
response = SimpleNamespace(
|
||||
status_code=403,
|
||||
text='{"status":"error"}',
|
||||
json=lambda: {
|
||||
"status": "error",
|
||||
"error": {
|
||||
"code": "PERMISSION_DENIED",
|
||||
"message": "direct host filesystem paths are not allowed",
|
||||
},
|
||||
},
|
||||
raise_for_status=lambda: None,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="PERMISSION_DENIED"):
|
||||
client._parse_response(response)
|
||||
|
||||
|
||||
def test_viking_client_headers_include_bearer_when_api_key_set():
|
||||
client = _VikingClient(
|
||||
"https://example.com",
|
||||
api_key="test-key",
|
||||
account="acct",
|
||||
user="usr",
|
||||
agent="hermes",
|
||||
)
|
||||
headers = client._headers()
|
||||
assert headers["X-API-Key"] == "test-key"
|
||||
assert headers["Authorization"] == "Bearer test-key"
|
||||
|
||||
|
||||
def test_viking_client_headers_send_tenant_when_default():
|
||||
# account/user set to the literal string "default". OpenViking 0.3.x
|
||||
# requires X-OpenViking-Account and X-OpenViking-User for ROOT API key
|
||||
# requests to tenant-scoped APIs — omitting them causes
|
||||
# INVALID_ARGUMENT errors even when account="default".
|
||||
client = _VikingClient(
|
||||
"https://example.com",
|
||||
api_key="test-key",
|
||||
account="default",
|
||||
user="default",
|
||||
agent="hermes",
|
||||
)
|
||||
headers = client._headers()
|
||||
assert headers["X-OpenViking-Account"] == "default"
|
||||
assert headers["X-OpenViking-User"] == "default"
|
||||
assert headers["X-OpenViking-Agent"] == "hermes"
|
||||
assert headers["Authorization"] == "Bearer test-key"
|
||||
|
||||
|
||||
def test_viking_client_headers_send_tenant_when_empty_falls_back_to_default():
|
||||
# Empty account/user strings fall back to "default" via the constructor.
|
||||
# Headers are sent even for the default value — ROOT API keys need them.
|
||||
client = _VikingClient(
|
||||
"https://example.com",
|
||||
api_key="",
|
||||
account="",
|
||||
user="",
|
||||
agent="hermes",
|
||||
)
|
||||
headers = client._headers()
|
||||
assert headers["X-OpenViking-Account"] == "default"
|
||||
assert headers["X-OpenViking-User"] == "default"
|
||||
assert "Authorization" not in headers
|
||||
assert "X-API-Key" not in headers
|
||||
|
||||
|
||||
def test_viking_client_headers_sent_with_real_tenant_values():
|
||||
client = _VikingClient(
|
||||
"https://example.com",
|
||||
api_key="test-key",
|
||||
account="real-account",
|
||||
user="real-user",
|
||||
agent="hermes",
|
||||
)
|
||||
headers = client._headers()
|
||||
assert headers["X-OpenViking-Account"] == "real-account"
|
||||
assert headers["X-OpenViking-User"] == "real-user"
|
||||
|
||||
|
||||
def test_viking_client_health_sends_auth_headers(monkeypatch):
|
||||
client = _VikingClient(
|
||||
"https://example.com",
|
||||
api_key="test-key",
|
||||
account="",
|
||||
user="",
|
||||
agent="hermes",
|
||||
)
|
||||
captured = {}
|
||||
|
||||
def capture_get(url, **kwargs):
|
||||
captured["url"] = url
|
||||
captured["headers"] = kwargs.get("headers") or {}
|
||||
return SimpleNamespace(status_code=200)
|
||||
|
||||
monkeypatch.setattr(client._httpx, "get", capture_get)
|
||||
assert client.health() is True
|
||||
assert captured["url"] == "https://example.com/health"
|
||||
assert captured["headers"]["Authorization"] == "Bearer test-key"
|
||||
@@ -0,0 +1,423 @@
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.memory.supermemory import (
|
||||
SupermemoryMemoryProvider,
|
||||
_clean_text_for_capture,
|
||||
_format_prefetch_context,
|
||||
_load_supermemory_config,
|
||||
_save_supermemory_config,
|
||||
)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, api_key: str, timeout: float, container_tag: str, search_mode: str = "hybrid"):
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
self.container_tag = container_tag
|
||||
self.search_mode = search_mode
|
||||
self.add_calls = []
|
||||
self.search_results = []
|
||||
self.profile_response = {"static": [], "dynamic": [], "search_results": []}
|
||||
self.ingest_calls = []
|
||||
self.forgotten_ids = []
|
||||
self.forget_by_query_response = {"success": True, "message": "Forgot"}
|
||||
|
||||
def add_memory(self, content, metadata=None, *, entity_context="",
|
||||
container_tag=None, custom_id=None):
|
||||
self.add_calls.append({
|
||||
"content": content,
|
||||
"metadata": metadata,
|
||||
"entity_context": entity_context,
|
||||
"container_tag": container_tag,
|
||||
"custom_id": custom_id,
|
||||
})
|
||||
return {"id": "mem_123"}
|
||||
|
||||
def search_memories(self, query, *, limit=5, container_tag=None, search_mode=None):
|
||||
return self.search_results
|
||||
|
||||
def get_profile(self, query=None, *, container_tag=None):
|
||||
return self.profile_response
|
||||
|
||||
def forget_memory(self, memory_id, *, container_tag=None):
|
||||
self.forgotten_ids.append(memory_id)
|
||||
|
||||
def forget_by_query(self, query, *, container_tag=None):
|
||||
return self.forget_by_query_response
|
||||
|
||||
def ingest_conversation(self, session_id, messages):
|
||||
self.ingest_calls.append({"session_id": session_id, "messages": messages})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("session-1", hermes_home=str(tmp_path), platform="cli")
|
||||
return p
|
||||
|
||||
|
||||
def test_is_available_false_without_api_key(monkeypatch):
|
||||
monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False)
|
||||
p = SupermemoryMemoryProvider()
|
||||
assert p.is_available() is False
|
||||
|
||||
|
||||
def test_is_available_false_when_import_missing(monkeypatch):
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
|
||||
import builtins
|
||||
real_import = builtins.__import__
|
||||
|
||||
def fake_import(name, *args, **kwargs):
|
||||
if name == "supermemory":
|
||||
raise ImportError("missing")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", fake_import)
|
||||
p = SupermemoryMemoryProvider()
|
||||
assert p.is_available() is False
|
||||
|
||||
|
||||
def test_load_and_save_config_round_trip(tmp_path):
|
||||
_save_supermemory_config({"container_tag": "demo-tag", "auto_capture": False}, str(tmp_path))
|
||||
cfg = _load_supermemory_config(str(tmp_path))
|
||||
# container_tag is kept raw — sanitization happens in initialize() after template resolution
|
||||
assert cfg["container_tag"] == "demo-tag"
|
||||
assert cfg["auto_capture"] is False
|
||||
assert cfg["auto_recall"] is True
|
||||
|
||||
|
||||
def test_clean_text_for_capture_strips_injected_context():
|
||||
text = "hello\n<supermemory-context>ignore me</supermemory-context>\nworld"
|
||||
assert _clean_text_for_capture(text) == "hello\nworld"
|
||||
|
||||
|
||||
def test_format_prefetch_context_deduplicates_overlap():
|
||||
result = _format_prefetch_context(
|
||||
static_facts=["Jordan prefers short answers"],
|
||||
dynamic_facts=["Jordan prefers short answers", "Uses Hermes"],
|
||||
search_results=[{"memory": "Uses Hermes", "similarity": 0.9}],
|
||||
max_results=10,
|
||||
)
|
||||
assert result.count("Jordan prefers short answers") == 1
|
||||
assert result.count("Uses Hermes") == 1
|
||||
assert "<supermemory-context>" in result
|
||||
|
||||
|
||||
def test_prefetch_includes_profile_on_first_turn(provider):
|
||||
provider._client.profile_response = {
|
||||
"static": ["Jordan prefers short answers"],
|
||||
"dynamic": ["Current project is Supermemory provider"],
|
||||
"search_results": [{"memory": "Working on Hermes memory provider", "similarity": 0.88}],
|
||||
}
|
||||
provider.on_turn_start(1, "start")
|
||||
result = provider.prefetch("what am I working on?")
|
||||
assert "User Profile (Persistent)" in result
|
||||
assert "Recent Context" in result
|
||||
assert "Relevant Memories" in result
|
||||
|
||||
|
||||
def test_prefetch_skips_profile_between_frequency(provider):
|
||||
provider._client.profile_response = {
|
||||
"static": ["Jordan prefers short answers"],
|
||||
"dynamic": ["Current project is Supermemory provider"],
|
||||
"search_results": [{"memory": "Working on Hermes memory provider", "similarity": 0.88}],
|
||||
}
|
||||
provider.on_turn_start(2, "next")
|
||||
result = provider.prefetch("what am I working on?")
|
||||
assert "Relevant Memories" in result
|
||||
assert "User Profile (Persistent)" not in result
|
||||
|
||||
|
||||
def test_sync_turn_skips_trivial_message(provider):
|
||||
provider.sync_turn("ok", "sure", session_id="session-1")
|
||||
assert provider._client.add_calls == []
|
||||
|
||||
|
||||
def test_sync_turn_persists_cleaned_exchange(provider):
|
||||
provider.sync_turn(
|
||||
"Please remember this\n<supermemory-context>ignore</supermemory-context>",
|
||||
"Got it, storing the context",
|
||||
session_id="session-1",
|
||||
)
|
||||
provider._sync_thread.join(timeout=1)
|
||||
assert len(provider._client.add_calls) == 1
|
||||
content = provider._client.add_calls[0]["content"]
|
||||
assert "ignore" not in content
|
||||
assert "[role: user]" in content
|
||||
assert "[role: assistant]" in content
|
||||
|
||||
|
||||
def test_on_session_end_ingests_clean_messages(provider):
|
||||
messages = [
|
||||
{"role": "system", "content": "skip"},
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": "hi there"},
|
||||
]
|
||||
provider.on_session_end(messages)
|
||||
assert len(provider._client.ingest_calls) == 1
|
||||
payload = provider._client.ingest_calls[0]
|
||||
assert payload["session_id"] == "session-1"
|
||||
assert payload["messages"] == [
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": "hi there"},
|
||||
]
|
||||
|
||||
|
||||
def test_on_memory_write_tracks_thread(provider):
|
||||
provider.on_memory_write("add", "memory", "Jordan likes concise docs")
|
||||
assert provider._write_thread is not None
|
||||
provider._write_thread.join(timeout=1)
|
||||
assert len(provider._client.add_calls) == 1
|
||||
assert provider._client.add_calls[0]["metadata"]["type"] == "explicit_memory"
|
||||
|
||||
|
||||
def test_shutdown_joins_and_clears_threads(provider, monkeypatch):
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_add_memory(content, metadata=None, *, entity_context="",
|
||||
container_tag=None, custom_id=None):
|
||||
started.set()
|
||||
release.wait(timeout=1)
|
||||
provider._client.add_calls.append({
|
||||
"content": content,
|
||||
"metadata": metadata,
|
||||
"entity_context": entity_context,
|
||||
})
|
||||
return {"id": "mem_slow"}
|
||||
|
||||
monkeypatch.setattr(provider._client, "add_memory", slow_add_memory)
|
||||
|
||||
provider.sync_turn(
|
||||
"Please remember this request in long-term memory",
|
||||
"Absolutely, I will keep that in long-term memory.",
|
||||
session_id="session-1",
|
||||
)
|
||||
assert started.wait(timeout=1)
|
||||
assert provider._sync_thread is not None
|
||||
|
||||
started.clear()
|
||||
provider.on_memory_write("add", "memory", "Jordan likes concise docs")
|
||||
assert started.wait(timeout=1)
|
||||
assert provider._write_thread is not None
|
||||
|
||||
release.set()
|
||||
provider.shutdown()
|
||||
|
||||
assert provider._sync_thread is None
|
||||
assert provider._write_thread is None
|
||||
assert provider._prefetch_thread is None
|
||||
assert len(provider._client.add_calls) == 2
|
||||
|
||||
|
||||
def test_store_tool_returns_saved_payload(provider):
|
||||
result = json.loads(provider.handle_tool_call("supermemory_store", {"content": "Jordan likes concise docs"}))
|
||||
assert result["saved"] is True
|
||||
assert result["id"] == "mem_123"
|
||||
|
||||
|
||||
def test_search_tool_formats_results(provider):
|
||||
provider._client.search_results = [
|
||||
{"id": "m1", "memory": "Jordan likes concise docs", "similarity": 0.92}
|
||||
]
|
||||
result = json.loads(provider.handle_tool_call("supermemory_search", {"query": "concise docs"}))
|
||||
assert result["count"] == 1
|
||||
assert result["results"][0]["similarity"] == 92
|
||||
|
||||
|
||||
def test_forget_tool_by_id(provider):
|
||||
result = json.loads(provider.handle_tool_call("supermemory_forget", {"id": "m1"}))
|
||||
assert result == {"forgotten": True, "id": "m1"}
|
||||
assert provider._client.forgotten_ids == ["m1"]
|
||||
|
||||
|
||||
def test_forget_tool_by_query(provider):
|
||||
provider._client.forget_by_query_response = {"success": True, "message": "Forgot one", "id": "m7"}
|
||||
result = json.loads(provider.handle_tool_call("supermemory_forget", {"query": "that thing"}))
|
||||
assert result["success"] is True
|
||||
assert result["id"] == "m7"
|
||||
|
||||
|
||||
def test_profile_tool_formats_sections(provider):
|
||||
provider._client.profile_response = {
|
||||
"static": ["Jordan prefers concise docs"],
|
||||
"dynamic": ["Working on Supermemory provider"],
|
||||
"search_results": [],
|
||||
}
|
||||
result = json.loads(provider.handle_tool_call("supermemory_profile", {}))
|
||||
assert result["static_count"] == 1
|
||||
assert result["dynamic_count"] == 1
|
||||
assert "User Profile (Persistent)" in result["profile"]
|
||||
|
||||
|
||||
def test_handle_tool_call_returns_error_when_unconfigured(monkeypatch):
|
||||
monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False)
|
||||
p = SupermemoryMemoryProvider()
|
||||
result = json.loads(p.handle_tool_call("supermemory_search", {"query": "x"}))
|
||||
assert "error" in result
|
||||
|
||||
|
||||
# -- Identity template tests --------------------------------------------------
|
||||
|
||||
|
||||
def test_identity_template_resolved_in_container_tag(monkeypatch, tmp_path):
|
||||
"""container_tag with {identity} resolves to profile-scoped tag."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({"container_tag": "hermes-{identity}"}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli", agent_identity="coder")
|
||||
assert p._container_tag == "hermes_coder"
|
||||
|
||||
|
||||
def test_identity_template_default_profile(monkeypatch, tmp_path):
|
||||
"""Without agent_identity kwarg, {identity} resolves to 'default'."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({"container_tag": "hermes-{identity}"}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
assert p._container_tag == "hermes_default"
|
||||
|
||||
|
||||
def test_container_tag_env_var_override(monkeypatch, tmp_path):
|
||||
"""SUPERMEMORY_CONTAINER_TAG env var overrides config."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setenv("SUPERMEMORY_CONTAINER_TAG", "env-override")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
assert p._container_tag == "env_override"
|
||||
|
||||
|
||||
# -- Search mode tests --------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_mode_config_passed_to_client(monkeypatch, tmp_path):
|
||||
"""search_mode from config is passed to _SupermemoryClient."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({"search_mode": "memories"}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
assert p._search_mode == "memories"
|
||||
assert p._client.search_mode == "memories"
|
||||
|
||||
|
||||
def test_invalid_search_mode_falls_back_to_default(monkeypatch, tmp_path):
|
||||
"""Invalid search_mode falls back to 'hybrid'."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({"search_mode": "invalid_mode"}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
assert p._search_mode == "hybrid"
|
||||
|
||||
|
||||
# -- Multi-container tests ----------------------------------------------------
|
||||
|
||||
|
||||
def test_multi_container_disabled_by_default(provider):
|
||||
"""Multi-container is off by default; schemas have no container_tag param."""
|
||||
assert provider._enable_custom_containers is False
|
||||
schemas = provider.get_tool_schemas()
|
||||
for s in schemas:
|
||||
assert "container_tag" not in s["parameters"]["properties"]
|
||||
|
||||
|
||||
def test_multi_container_enabled_adds_schema_param(monkeypatch, tmp_path):
|
||||
"""When enabled, tool schemas include container_tag parameter."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({
|
||||
"enable_custom_container_tags": True,
|
||||
"custom_containers": ["project-alpha", "shared"],
|
||||
}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
assert p._enable_custom_containers is True
|
||||
assert p._allowed_containers == ["hermes", "project_alpha", "shared"]
|
||||
schemas = p.get_tool_schemas()
|
||||
for s in schemas:
|
||||
assert "container_tag" in s["parameters"]["properties"]
|
||||
|
||||
|
||||
def test_multi_container_tool_store_with_custom_tag(monkeypatch, tmp_path):
|
||||
"""supermemory_store uses the resolved container_tag when multi-container is enabled."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({
|
||||
"enable_custom_container_tags": True,
|
||||
"custom_containers": ["project-alpha"],
|
||||
}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
result = json.loads(p.handle_tool_call("supermemory_store", {
|
||||
"content": "test memory",
|
||||
"container_tag": "project-alpha",
|
||||
}))
|
||||
assert result["saved"] is True
|
||||
assert result["container_tag"] == "project_alpha"
|
||||
assert p._client.add_calls[-1]["container_tag"] == "project_alpha"
|
||||
|
||||
|
||||
def test_multi_container_rejects_unlisted_tag(monkeypatch, tmp_path):
|
||||
"""Tool calls with a non-whitelisted container_tag return an error."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({
|
||||
"enable_custom_container_tags": True,
|
||||
"custom_containers": ["allowed-tag"],
|
||||
}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
result = json.loads(p.handle_tool_call("supermemory_store", {
|
||||
"content": "test",
|
||||
"container_tag": "forbidden-tag",
|
||||
}))
|
||||
assert "error" in result
|
||||
assert "not allowed" in result["error"]
|
||||
|
||||
|
||||
def test_multi_container_system_prompt_includes_instructions(monkeypatch, tmp_path):
|
||||
"""system_prompt_block includes container list and instructions when multi-container is enabled."""
|
||||
monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key")
|
||||
monkeypatch.setattr("plugins.memory.supermemory._SupermemoryClient", FakeClient)
|
||||
_save_supermemory_config({
|
||||
"enable_custom_container_tags": True,
|
||||
"custom_containers": ["docs"],
|
||||
"custom_container_instructions": "Use docs for documentation context.",
|
||||
}, str(tmp_path))
|
||||
p = SupermemoryMemoryProvider()
|
||||
p.initialize("s1", hermes_home=str(tmp_path), platform="cli")
|
||||
block = p.system_prompt_block()
|
||||
assert "Multi-container mode enabled" in block
|
||||
assert "docs" in block
|
||||
assert "Use docs for documentation context." in block
|
||||
|
||||
|
||||
def test_get_config_schema_minimal():
|
||||
"""get_config_schema only returns the API key field."""
|
||||
p = SupermemoryMemoryProvider()
|
||||
schema = p.get_config_schema()
|
||||
assert len(schema) == 1
|
||||
assert schema[0]["key"] == "api_key"
|
||||
assert schema[0]["secret"] is True
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode bits not enforced on Windows")
|
||||
def test_save_config_sets_owner_only_permissions(tmp_path):
|
||||
"""supermemory.json must be written with 0o600 so API key is not world-readable."""
|
||||
_save_supermemory_config({"api_key": "sm-test-key"}, str(tmp_path))
|
||||
config_file = tmp_path / "supermemory.json"
|
||||
assert config_file.exists()
|
||||
mode = stat.S_IMODE(config_file.stat().st_mode)
|
||||
assert mode == 0o600, f"Expected 0o600 (owner-only), got {oct(mode)}"
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Unit tests for the DeepSeek provider profile's thinking-mode wiring.
|
||||
|
||||
DeepSeek V4 (and the legacy ``deepseek-reasoner``) expects every request to
|
||||
carry an explicit ``extra_body.thinking`` parameter. Omitting it makes the
|
||||
server default to thinking-mode ON, which then enforces the
|
||||
``reasoning_content``-must-be-echoed-back contract on subsequent turns and
|
||||
breaks the conversation with HTTP 400 (#15700, #17212, #17825).
|
||||
|
||||
These tests pin the profile's wire-shape contract so DeepSeek requests stay
|
||||
correctly shaped without going live.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deepseek_profile():
|
||||
"""Resolve the registered DeepSeek profile.
|
||||
|
||||
Going through ``providers.get_provider_profile`` keeps the test honest —
|
||||
if someone later replaces the registered class with a plain
|
||||
``ProviderProfile``, every assertion below collapses.
|
||||
"""
|
||||
# ``model_tools`` triggers plugin discovery on import, which is what
|
||||
# registers the DeepSeek profile in the global provider registry.
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("deepseek")
|
||||
assert profile is not None, "deepseek provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestDeepSeekThinkingWireShape:
|
||||
"""``build_api_kwargs_extras`` produces DeepSeek's exact wire format."""
|
||||
|
||||
def test_v4_pro_default_enables_thinking_without_effort(self, deepseek_profile):
|
||||
"""No reasoning_config → thinking enabled, server picks default effort."""
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model="deepseek-v4-pro"
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_v4_pro_enabled_with_high_effort(self, deepseek_profile):
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
|
||||
def test_standard_efforts_pass_through(self, deepseek_profile, effort):
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
|
||||
def test_xhigh_and_max_normalize_to_max(self, deepseek_profile, effort):
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
def test_explicitly_disabled_sends_disabled_marker(self, deepseek_profile):
|
||||
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
|
||||
|
||||
The crucial bit is that the parameter is *sent* at all — DeepSeek
|
||||
defaults to thinking-on when ``thinking`` is absent.
|
||||
"""
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False}, model="deepseek-v4-pro"
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
# No effort when disabled — DeepSeek rejects it.
|
||||
assert top_level == {}
|
||||
|
||||
def test_disabled_ignores_effort_field(self, deepseek_profile):
|
||||
"""Effort silently dropped when thinking is off."""
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False, "effort": "high"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
def test_unknown_effort_omits_top_level(self, deepseek_profile):
|
||||
"""Garbage effort → omit reasoning_effort so DeepSeek applies its default."""
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "garbage"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
def test_empty_effort_omits_top_level(self, deepseek_profile):
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": ""},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestDeepSeekModelGating:
|
||||
"""V4 family + ``deepseek-reasoner`` get thinking; V3 stays untouched."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"deepseek-v4-pro",
|
||||
"deepseek-v4-flash",
|
||||
"deepseek-v4-future-variant",
|
||||
"deepseek-reasoner",
|
||||
"DEEPSEEK-V4-PRO", # case-insensitive
|
||||
],
|
||||
)
|
||||
def test_thinking_capable_models_emit_thinking(self, deepseek_profile, model):
|
||||
extra_body, _ = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model=model
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"deepseek-chat", # V3 alias
|
||||
"deepseek-v3-0324", # explicit V3
|
||||
"deepseek-v3.1", # V3 minor revisions
|
||||
"", # bare/unknown
|
||||
None, # missing
|
||||
"deepseek-unknown", # unrecognized
|
||||
],
|
||||
)
|
||||
def test_non_thinking_models_emit_nothing(self, deepseek_profile, model):
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"}, model=model
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestDeepSeekFullKwargsIntegration:
|
||||
"""End-to-end: the transport's full kwargs match DeepSeek's live wire format.
|
||||
|
||||
The live test harness in ``tests/run_agent/test_deepseek_v4_thinking_live.py``
|
||||
sends ``{"reasoning_effort": "high", "extra_body": {"thinking": {"type":
|
||||
"enabled"}}}``. Confirm the transport produces that exact shape when wired
|
||||
through the registered DeepSeek profile.
|
||||
"""
|
||||
|
||||
def test_full_kwargs_match_live_wire_shape(self, deepseek_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-v4-pro",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=deepseek_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://api.deepseek.com/v1",
|
||||
provider_name="deepseek",
|
||||
)
|
||||
assert kwargs["model"] == "deepseek-v4-pro"
|
||||
assert kwargs["reasoning_effort"] == "high"
|
||||
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
|
||||
|
||||
def test_v3_chat_full_kwargs_omit_thinking(self, deepseek_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-chat",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=deepseek_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://api.deepseek.com/v1",
|
||||
provider_name="deepseek",
|
||||
)
|
||||
assert "reasoning_effort" not in kwargs
|
||||
assert "extra_body" not in kwargs or "thinking" not in kwargs.get("extra_body", {})
|
||||
|
||||
|
||||
class TestDeepSeekAuxModel:
|
||||
"""DeepSeek aux model is set on the profile so users stop seeing the
|
||||
bogus 'No auxiliary LLM provider configured' warning (#26924).
|
||||
|
||||
Pinned at the profile layer rather than the legacy
|
||||
`_API_KEY_PROVIDER_AUX_MODELS_FALLBACK` dict — new providers are
|
||||
expected to set `default_aux_model` on `ProviderProfile`, and the
|
||||
fallback dict only exists for providers that predate the profiles
|
||||
system.
|
||||
"""
|
||||
|
||||
def test_profile_advertises_deepseek_chat(self, deepseek_profile):
|
||||
assert deepseek_profile.default_aux_model == "deepseek-chat"
|
||||
|
||||
def test_consumer_api_returns_deepseek_chat(self):
|
||||
from agent.auxiliary_client import _get_aux_model_for_provider
|
||||
assert _get_aux_model_for_provider("deepseek") == "deepseek-chat"
|
||||
|
||||
def test_consumer_api_returns_non_empty(self):
|
||||
from agent.auxiliary_client import _get_aux_model_for_provider
|
||||
assert _get_aux_model_for_provider("deepseek") != ""
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Unit tests for OpenCode Go reasoning-control wiring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opencode_go_profile():
|
||||
"""Resolve the registered OpenCode Go provider profile."""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("opencode-go")
|
||||
assert profile is not None, "opencode-go provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestOpenCodeGoKimiReasoning:
|
||||
"""Kimi K2 models use Moonshot's thinking + reasoning_effort shape on OpenCode Go."""
|
||||
|
||||
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_minimal_effort_enables_thinking_without_effort(self, opencode_go_profile):
|
||||
# "minimal" is not a Moonshot-supported value — drop it, keep thinking on.
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "minimal"},
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort",
|
||||
[
|
||||
"xhigh",
|
||||
"max",
|
||||
],
|
||||
)
|
||||
def test_strong_efforts_clamp_to_high(self, opencode_go_profile, effort):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="moonshotai/kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
def test_low_and_medium_pass_through(self, opencode_go_profile):
|
||||
for effort in ("low", "medium"):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="kimi-k2.5",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
def test_no_config_preserves_server_default(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None,
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOpenCodeGoDeepSeekThinking:
|
||||
"""DeepSeek V4 models use DeepSeek-style thinking controls on OpenCode Go."""
|
||||
|
||||
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False, "effort": "high"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_no_config_emits_thinking_enabled_without_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None,
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_minimal_effort_enables_thinking_without_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "minimal"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_xhigh_and_max_normalize_to_max(self, opencode_go_profile):
|
||||
for effort in ("xhigh", "max"):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
|
||||
class TestOpenCodeGoModelGating:
|
||||
"""Other OpenCode Go models must not receive Kimi/DeepSeek controls."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"glm-5.1",
|
||||
"qwen3.6-plus",
|
||||
"minimax-m2.7",
|
||||
"deepseek-v3.1",
|
||||
"deepseek-chat",
|
||||
"",
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_non_target_models_emit_nothing(self, opencode_go_profile, model):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model=model,
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOpenCodeGoFullKwargsIntegration:
|
||||
"""End-to-end transport kwargs include the profile-provided controls."""
|
||||
|
||||
def test_kimi_reasoning_reaches_extra_body_and_top_level(self, opencode_go_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="kimi-k2.6",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=opencode_go_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
)
|
||||
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
|
||||
assert kwargs["reasoning_effort"] == "high"
|
||||
|
||||
def test_deepseek_thinking_reaches_extra_body_and_top_level(
|
||||
self, opencode_go_profile
|
||||
):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-v4-pro",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=opencode_go_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
)
|
||||
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
|
||||
assert kwargs["reasoning_effort"] == "high"
|
||||
@@ -0,0 +1,378 @@
|
||||
"""Tests for the bundled hermes-achievements dashboard plugin.
|
||||
|
||||
These target the two behaviors that matter for official integration:
|
||||
|
||||
* The 200-session scan cap is removed — the plugin now walks the entire
|
||||
session history by default. Lifetime badges (tens of thousands of
|
||||
tool calls) were unreachable before this fix on long-running installs.
|
||||
* First-ever scans run in a background thread so the dashboard request
|
||||
path never blocks, even on 8000+ session databases where a cold scan
|
||||
takes minutes.
|
||||
|
||||
The upstream repo ships its own unittest suite under
|
||||
``plugins/hermes-achievements/tests/`` covering the achievement engine
|
||||
internals (tier math, secret-state handling, catalog invariants). These
|
||||
tests live at the hermes-agent level and focus on the integration
|
||||
contract: the plugin scans ALL of your sessions, not the first 200.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
PLUGIN_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "plugins"
|
||||
/ "hermes-achievements"
|
||||
/ "dashboard"
|
||||
/ "plugin_api.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plugin_api(tmp_path, monkeypatch):
|
||||
"""Load plugin_api with isolated ~/.hermes so state/snapshot files don't collide.
|
||||
|
||||
We load the module fresh per test because the plugin keeps module-level
|
||||
caches (``_SNAPSHOT_CACHE``, ``_SCAN_STATUS``, background thread handle).
|
||||
Reloading gives each test a clean world.
|
||||
"""
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
f"plugin_api_test_{id(tmp_path)}", PLUGIN_MODULE_PATH
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
# Stash monkeypatch so ``_install_fake_session_db`` can use it to
|
||||
# swap ``sys.modules['hermes_state']`` with auto-restoration. Without
|
||||
# this, a raw ``sys.modules[...] = fake`` assignment would leak the
|
||||
# fake into later tests in the same xdist worker — breaking every
|
||||
# test that does ``from hermes_state import SessionDB``.
|
||||
module._test_monkeypatch = monkeypatch
|
||||
yield module
|
||||
|
||||
|
||||
class _FakeSessionDB:
|
||||
"""Stand-in for hermes_state.SessionDB that records scan calls."""
|
||||
|
||||
def __init__(self, session_count: int, scan_delay: float = 0):
|
||||
self.session_count = session_count
|
||||
self.scan_delay = scan_delay
|
||||
self.last_limit: Optional[int] = None
|
||||
self.last_include_children: Optional[bool] = None
|
||||
self.list_calls = 0
|
||||
self.messages_calls = 0
|
||||
|
||||
def list_sessions_rich(
|
||||
self,
|
||||
source: Optional[str] = None,
|
||||
exclude_sources: Optional[List[str]] = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
include_children: bool = False,
|
||||
project_compression_tips: bool = True,
|
||||
) -> List[Dict[str, Any]]:
|
||||
if self.scan_delay:
|
||||
time.sleep(self.scan_delay)
|
||||
self.last_limit = limit
|
||||
self.last_include_children = include_children
|
||||
self.list_calls += 1
|
||||
# SQLite semantics: LIMIT -1 = unlimited. Honor that here.
|
||||
effective = self.session_count if limit == -1 else min(self.session_count, limit)
|
||||
now = int(time.time())
|
||||
return [
|
||||
{
|
||||
"id": f"sess-{i}",
|
||||
"title": f"Session {i}",
|
||||
"preview": f"preview {i}",
|
||||
"started_at": now - (self.session_count - i) * 60,
|
||||
"last_active": now - (self.session_count - i) * 60 + 30,
|
||||
"source": "cli",
|
||||
"model": "test-model",
|
||||
}
|
||||
for i in range(effective)
|
||||
]
|
||||
|
||||
def get_messages(self, session_id: str) -> List[Dict[str, Any]]:
|
||||
self.messages_calls += 1
|
||||
return [
|
||||
{"role": "user", "content": f"ask {session_id}"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [{"function": {"name": "terminal"}}],
|
||||
},
|
||||
{"role": "tool", "tool_name": "terminal", "content": "ok"},
|
||||
]
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _install_fake_session_db(plugin_api, fake_db):
|
||||
"""Inject a fake SessionDB so ``scan_sessions`` finds it via its local import.
|
||||
|
||||
Uses the monkeypatch stashed on ``plugin_api`` by the fixture, so the
|
||||
``sys.modules['hermes_state']`` swap is auto-restored at test teardown
|
||||
and cannot leak into unrelated tests in the same xdist worker.
|
||||
"""
|
||||
fake_module = type(sys)("hermes_state")
|
||||
fake_module.SessionDB = lambda: fake_db
|
||||
plugin_api._test_monkeypatch.setitem(sys.modules, "hermes_state", fake_module)
|
||||
|
||||
|
||||
def test_scan_sessions_default_scans_all_history_not_first_200(plugin_api):
|
||||
"""Bug regression: ``scan_sessions()`` used to cap at limit=200.
|
||||
|
||||
A user with 8000+ sessions would only see ~2% of their history in
|
||||
achievement totals, making lifetime badges unreachable. The default
|
||||
now passes ``LIMIT -1`` (SQLite "unlimited") to ``list_sessions_rich``.
|
||||
"""
|
||||
fake_db = _FakeSessionDB(session_count=500) # > old 200 cap
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
result = plugin_api.scan_sessions()
|
||||
|
||||
assert fake_db.last_limit == -1, (
|
||||
"scan_sessions() must pass LIMIT=-1 (unlimited) to list_sessions_rich "
|
||||
f"by default, got {fake_db.last_limit}"
|
||||
)
|
||||
assert fake_db.last_include_children is True, (
|
||||
"scan_sessions() must include subagent/compression child sessions so "
|
||||
"tool calls made in delegated agents still count toward achievements"
|
||||
)
|
||||
assert len(result["sessions"]) == 500
|
||||
assert result["scan_meta"]["sessions_total"] == 500
|
||||
|
||||
|
||||
def test_scan_sessions_explicit_positive_limit_is_honored(plugin_api):
|
||||
"""Callers can still pass a small limit for smoke tests."""
|
||||
fake_db = _FakeSessionDB(session_count=500)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
result = plugin_api.scan_sessions(limit=10)
|
||||
|
||||
assert fake_db.last_limit == 10
|
||||
assert len(result["sessions"]) == 10
|
||||
|
||||
|
||||
def test_scan_sessions_zero_or_negative_limit_means_unlimited(plugin_api):
|
||||
"""``limit=0`` and ``limit=-1`` both map to the unlimited path."""
|
||||
fake_db = _FakeSessionDB(session_count=300)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
plugin_api.scan_sessions(limit=0)
|
||||
assert fake_db.last_limit == -1
|
||||
|
||||
plugin_api.scan_sessions(limit=-1)
|
||||
assert fake_db.last_limit == -1
|
||||
|
||||
|
||||
def test_evaluate_all_first_run_returns_pending_and_starts_background_scan(plugin_api):
|
||||
"""First-ever evaluate_all with no cache returns a pending placeholder
|
||||
immediately and kicks off a background scan thread. Cold scans on
|
||||
large DBs take minutes — blocking the dashboard request path is not
|
||||
acceptable.
|
||||
"""
|
||||
fake_db = _FakeSessionDB(session_count=50)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
# Wrap _run_scan_and_update_cache so we can release it on demand,
|
||||
# simulating a slow cold scan without actually waiting.
|
||||
scan_started = threading.Event()
|
||||
allow_scan_finish = threading.Event()
|
||||
original_run = plugin_api._run_scan_and_update_cache
|
||||
|
||||
def gated_run(*args, **kwargs):
|
||||
scan_started.set()
|
||||
allow_scan_finish.wait(timeout=5)
|
||||
original_run(*args, **kwargs)
|
||||
|
||||
plugin_api._run_scan_and_update_cache = gated_run
|
||||
|
||||
t0 = time.time()
|
||||
result = plugin_api.evaluate_all()
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# Immediate return — should not block waiting for the scan.
|
||||
assert elapsed < 1.0, f"evaluate_all blocked for {elapsed:.2f}s on first run"
|
||||
assert result["scan_meta"]["mode"] == "pending"
|
||||
assert result["unlocked_count"] == 0
|
||||
# Catalog still rendered so UI has something to draw.
|
||||
assert result["total_count"] >= 60
|
||||
|
||||
# Background scan is running.
|
||||
assert scan_started.wait(timeout=2), "background scan did not start"
|
||||
|
||||
# Let the scan complete, then a second call returns real data.
|
||||
allow_scan_finish.set()
|
||||
# Wait for thread to finish.
|
||||
thread = plugin_api._BACKGROUND_SCAN_THREAD
|
||||
assert thread is not None
|
||||
thread.join(timeout=5)
|
||||
assert not thread.is_alive()
|
||||
|
||||
second = plugin_api.evaluate_all()
|
||||
assert second["scan_meta"]["mode"] != "pending"
|
||||
assert second["scan_meta"].get("sessions_total") == 50
|
||||
|
||||
|
||||
def test_evaluate_all_stale_cache_serves_stale_and_refreshes_in_background(plugin_api):
|
||||
"""When the snapshot is on-disk but older than TTL, evaluate_all returns
|
||||
the stale data immediately and kicks a background refresh. Users don't
|
||||
stare at a loading spinner every time TTL expires.
|
||||
"""
|
||||
fake_db = _FakeSessionDB(session_count=10, scan_delay=2.0)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
stale_generated_at = int(time.time()) - plugin_api.SNAPSHOT_TTL_SECONDS - 60
|
||||
stale_payload = {
|
||||
"achievements": [],
|
||||
"sessions": [],
|
||||
"aggregate": {},
|
||||
"scan_meta": {"mode": "full", "sessions_total": 1, "sessions_rescanned": 1, "sessions_reused": 0},
|
||||
"error": None,
|
||||
"unlocked_count": 0,
|
||||
"discovered_count": 0,
|
||||
"secret_count": 0,
|
||||
"total_count": 0,
|
||||
"generated_at": stale_generated_at,
|
||||
}
|
||||
plugin_api.save_snapshot(stale_payload)
|
||||
|
||||
t0 = time.time()
|
||||
result = plugin_api.evaluate_all()
|
||||
elapsed = time.time() - t0
|
||||
|
||||
assert elapsed < 1.0, f"evaluate_all blocked for {elapsed:.2f}s serving stale data"
|
||||
assert result["generated_at"] == stale_generated_at
|
||||
|
||||
# Background scan should be running or have completed.
|
||||
thread = plugin_api._BACKGROUND_SCAN_THREAD
|
||||
assert thread is not None
|
||||
thread.join(timeout=5)
|
||||
|
||||
fresh = plugin_api.evaluate_all()
|
||||
assert fresh["generated_at"] >= stale_generated_at
|
||||
|
||||
|
||||
def test_evaluate_all_force_runs_synchronously(plugin_api):
|
||||
"""Manual /rescan (force=True) blocks the caller — users clicking
|
||||
the rescan button expect up-to-date data when the call returns.
|
||||
"""
|
||||
fake_db = _FakeSessionDB(session_count=25)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
result = plugin_api.evaluate_all(force=True)
|
||||
|
||||
# Synchronous — snapshot is fresh on return.
|
||||
assert result["scan_meta"].get("sessions_total") == 25
|
||||
assert result["scan_meta"]["mode"] in {"full", "incremental"}
|
||||
|
||||
|
||||
def test_start_background_scan_is_idempotent_while_running(plugin_api):
|
||||
"""Multiple concurrent dashboard requests must not spawn duplicate scans."""
|
||||
fake_db = _FakeSessionDB(session_count=5)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
release = threading.Event()
|
||||
original_run = plugin_api._run_scan_and_update_cache
|
||||
|
||||
def gated_run(*args, **kwargs):
|
||||
release.wait(timeout=5)
|
||||
original_run(*args, **kwargs)
|
||||
|
||||
plugin_api._run_scan_and_update_cache = gated_run
|
||||
|
||||
plugin_api._start_background_scan()
|
||||
first_thread = plugin_api._BACKGROUND_SCAN_THREAD
|
||||
assert first_thread is not None and first_thread.is_alive()
|
||||
|
||||
plugin_api._start_background_scan()
|
||||
plugin_api._start_background_scan()
|
||||
|
||||
assert plugin_api._BACKGROUND_SCAN_THREAD is first_thread
|
||||
|
||||
release.set()
|
||||
first_thread.join(timeout=5)
|
||||
|
||||
|
||||
def test_background_scan_publishes_partial_snapshots(plugin_api):
|
||||
"""The background scanner publishes intermediate snapshots to the cache
|
||||
every ~N sessions. Each dashboard refresh during a long cold scan sees
|
||||
more badges unlocked instead of staring at zeros for minutes and then
|
||||
having everything pop at the end.
|
||||
"""
|
||||
fake_db = _FakeSessionDB(session_count=750)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
# Record every partial snapshot the scanner publishes.
|
||||
partial_snapshots: List[Dict[str, Any]] = []
|
||||
original_compute_from_scan = plugin_api._compute_from_scan
|
||||
|
||||
def recording_compute(scan, *, is_partial=False):
|
||||
result = original_compute_from_scan(scan, is_partial=is_partial)
|
||||
if is_partial:
|
||||
partial_snapshots.append(result)
|
||||
return result
|
||||
|
||||
plugin_api._compute_from_scan = recording_compute
|
||||
|
||||
# scan 750 sessions with progress_every=250 → expect 2 intermediate
|
||||
# publications (at 250 and 500; the final 750 call goes through the
|
||||
# finished, non-partial path).
|
||||
plugin_api._run_scan_and_update_cache(publish_partial_snapshots=True)
|
||||
|
||||
assert len(partial_snapshots) >= 2, (
|
||||
f"expected at least 2 partial publications on a 750-session scan with "
|
||||
f"progress_every=250, got {len(partial_snapshots)}"
|
||||
)
|
||||
# Partial snapshots should report growing session counts.
|
||||
counts = [p["scan_meta"].get("sessions_scanned_so_far") for p in partial_snapshots]
|
||||
assert counts == sorted(counts), f"partial session counts not monotonic: {counts}"
|
||||
assert counts[0] < 750 and counts[-1] < 750, (
|
||||
f"partial counts should be less than the final total; got {counts}"
|
||||
)
|
||||
# Every partial reports the expected end-state total so the UI can
|
||||
# show an accurate progress bar.
|
||||
for p in partial_snapshots:
|
||||
assert p["scan_meta"].get("sessions_expected_total") == 750
|
||||
|
||||
# Final snapshot in cache is the real (non-partial) one.
|
||||
final = plugin_api._SNAPSHOT_CACHE
|
||||
assert final is not None
|
||||
assert final["scan_meta"].get("mode") != "in_progress"
|
||||
assert final["scan_meta"].get("sessions_total") == 750
|
||||
|
||||
|
||||
def test_partial_snapshots_do_not_persist_unlock_timestamps(plugin_api):
|
||||
"""Intermediate snapshots must not write to state.json — an unlock
|
||||
that appears at 30% scan progress could disappear when a later session
|
||||
rebalances the aggregate. Only the final snapshot records ``unlocked_at``.
|
||||
"""
|
||||
fake_db = _FakeSessionDB(session_count=10)
|
||||
_install_fake_session_db(plugin_api, fake_db)
|
||||
|
||||
# Seed empty state, then invoke partial compute directly.
|
||||
plugin_api.save_state({"unlocks": {}})
|
||||
partial_scan = {
|
||||
"sessions": [{"session_id": "x", "tool_call_count": 99999, "tool_names": set()}],
|
||||
"aggregate": {"max_tool_calls_in_session": 99999, "total_tool_calls": 99999},
|
||||
"scan_meta": {"mode": "in_progress"},
|
||||
}
|
||||
result = plugin_api._compute_from_scan(partial_scan, is_partial=True)
|
||||
|
||||
# Some achievements should evaluate as unlocked in this aggregate...
|
||||
assert any(a["unlocked"] for a in result["achievements"])
|
||||
|
||||
# ...but state.json on disk stays empty (no timestamps were recorded).
|
||||
persisted = plugin_api.load_state()
|
||||
assert persisted.get("unlocks", {}) == {}, (
|
||||
"partial scans must not record unlock timestamps — a later session "
|
||||
"could change whether the badge deserves to be unlocked yet"
|
||||
)
|
||||
@@ -0,0 +1,455 @@
|
||||
"""Tests for the disk-cleanup plugin.
|
||||
|
||||
Covers the bundled plugin at ``plugins/disk-cleanup/``:
|
||||
|
||||
* ``disk_cleanup`` library: track / forget / dry_run / quick / status,
|
||||
``is_safe_path`` and ``guess_category`` filtering.
|
||||
* Plugin ``__init__``: ``post_tool_call`` hook auto-tracks files created
|
||||
by ``write_file`` / ``terminal``; ``on_session_end`` hook runs quick
|
||||
cleanup when anything was tracked during the turn.
|
||||
* Slash command handler: status / dry-run / quick / track / forget /
|
||||
unknown subcommand behaviours.
|
||||
* Bundled-plugin discovery via ``PluginManager.discover_and_load``.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(tmp_path, monkeypatch):
|
||||
"""Isolate HERMES_HOME for each test.
|
||||
|
||||
The global hermetic fixture already redirects HERMES_HOME to a tempdir,
|
||||
but we want the plugin to work with a predictable subpath. We reset
|
||||
HERMES_HOME here for clarity.
|
||||
"""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
yield hermes_home
|
||||
|
||||
|
||||
def _load_lib():
|
||||
"""Import the plugin's library module directly from the repo path."""
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
lib_path = repo_root / "plugins" / "disk-cleanup" / "disk_cleanup.py"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"disk_cleanup_under_test", lib_path
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def _load_plugin_init():
|
||||
"""Import the plugin's __init__.py (which depends on the library)."""
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
plugin_dir = repo_root / "plugins" / "disk-cleanup"
|
||||
# Use the PluginManager's module naming convention so relative imports work.
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"hermes_plugins.disk_cleanup",
|
||||
plugin_dir / "__init__.py",
|
||||
submodule_search_locations=[str(plugin_dir)],
|
||||
)
|
||||
# Ensure parent namespace package exists for the relative `. import disk_cleanup`
|
||||
import types
|
||||
if "hermes_plugins" not in sys.modules:
|
||||
ns = types.ModuleType("hermes_plugins")
|
||||
ns.__path__ = []
|
||||
sys.modules["hermes_plugins"] = ns
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
mod.__package__ = "hermes_plugins.disk_cleanup"
|
||||
mod.__path__ = [str(plugin_dir)]
|
||||
sys.modules["hermes_plugins.disk_cleanup"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Library tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestIsSafePath:
|
||||
def test_accepts_path_under_hermes_home(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "subdir" / "file.txt"
|
||||
p.parent.mkdir()
|
||||
p.write_text("x")
|
||||
assert dg.is_safe_path(p) is True
|
||||
|
||||
def test_rejects_outside_hermes_home(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
assert dg.is_safe_path(Path("/etc/passwd")) is False
|
||||
|
||||
def test_accepts_tmp_hermes_prefix(self, _isolate_env, tmp_path):
|
||||
dg = _load_lib()
|
||||
assert dg.is_safe_path(Path("/tmp/hermes-abc/x.log")) is True
|
||||
|
||||
def test_rejects_plain_tmp(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
assert dg.is_safe_path(Path("/tmp/other.log")) is False
|
||||
|
||||
def test_rejects_windows_mount(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
assert dg.is_safe_path(Path("/mnt/c/Users/x/test.txt")) is False
|
||||
|
||||
|
||||
class TestGuessCategory:
|
||||
def test_test_prefix(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "test_foo.py"
|
||||
p.write_text("x")
|
||||
assert dg.guess_category(p) == "test"
|
||||
|
||||
def test_tmp_prefix(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "tmp_foo.log"
|
||||
p.write_text("x")
|
||||
assert dg.guess_category(p) == "test"
|
||||
|
||||
def test_dot_test_suffix(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "mything.test.js"
|
||||
p.write_text("x")
|
||||
assert dg.guess_category(p) == "test"
|
||||
|
||||
def test_skips_protected_top_level(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
logs_dir = _isolate_env / "logs"
|
||||
logs_dir.mkdir()
|
||||
p = logs_dir / "test_log.txt"
|
||||
p.write_text("x")
|
||||
# Even though it matches test_* pattern, logs/ is excluded.
|
||||
assert dg.guess_category(p) is None
|
||||
|
||||
def test_cron_subtree_categorised(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
# Only files under ``cron/output/`` are disposable run artifacts.
|
||||
output_dir = _isolate_env / "cron" / "output" / "job_123"
|
||||
output_dir.mkdir(parents=True)
|
||||
p = output_dir / "run.md"
|
||||
p.write_text("x")
|
||||
assert dg.guess_category(p) == "cron-output"
|
||||
|
||||
def test_cron_jobs_json_not_tracked(self, _isolate_env):
|
||||
"""Regression for #32164: the cron registry must never be tracked."""
|
||||
dg = _load_lib()
|
||||
cron_dir = _isolate_env / "cron"
|
||||
cron_dir.mkdir()
|
||||
p = cron_dir / "jobs.json"
|
||||
p.write_text("[]")
|
||||
assert dg.guess_category(p) is None
|
||||
|
||||
def test_cron_tick_lock_not_tracked(self, _isolate_env):
|
||||
"""Regression for #32164: cron tick-lock is control-plane state."""
|
||||
dg = _load_lib()
|
||||
cron_dir = _isolate_env / "cron"
|
||||
cron_dir.mkdir()
|
||||
p = cron_dir / ".tick.lock"
|
||||
p.write_text("")
|
||||
assert dg.guess_category(p) is None
|
||||
|
||||
def test_cronjobs_top_level_not_tracked(self, _isolate_env):
|
||||
"""The legacy ``cronjobs`` alias is also control-plane at the top."""
|
||||
dg = _load_lib()
|
||||
cron_dir = _isolate_env / "cronjobs"
|
||||
cron_dir.mkdir()
|
||||
p = cron_dir / "jobs.json"
|
||||
p.write_text("[]")
|
||||
assert dg.guess_category(p) is None
|
||||
|
||||
def test_ordinary_file_returns_none(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "notes.md"
|
||||
p.write_text("x")
|
||||
assert dg.guess_category(p) is None
|
||||
|
||||
|
||||
class TestTrackForgetQuick:
|
||||
def test_track_then_quick_deletes_test(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "test_a.py"
|
||||
p.write_text("x")
|
||||
assert dg.track(str(p), "test", silent=True) is True
|
||||
summary = dg.quick()
|
||||
assert summary["deleted"] == 1
|
||||
assert not p.exists()
|
||||
|
||||
def test_track_dedup(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "test_a.py"
|
||||
p.write_text("x")
|
||||
assert dg.track(str(p), "test", silent=True) is True
|
||||
# Second call returns False (already tracked)
|
||||
assert dg.track(str(p), "test", silent=True) is False
|
||||
|
||||
def test_track_rejects_outside_home(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
# /etc/hostname exists on most Linux boxes; fall back if not.
|
||||
outside = "/etc/hostname" if Path("/etc/hostname").exists() else "/etc/passwd"
|
||||
assert dg.track(outside, "test", silent=True) is False
|
||||
|
||||
def test_track_skips_missing(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
assert dg.track(str(_isolate_env / "nope.txt"), "test", silent=True) is False
|
||||
|
||||
def test_forget_removes_entry(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "keep.tmp"
|
||||
p.write_text("x")
|
||||
dg.track(str(p), "temp", silent=True)
|
||||
assert dg.forget(str(p)) == 1
|
||||
assert p.exists() # forget does NOT delete the file
|
||||
|
||||
def test_quick_preserves_unexpired_temp(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "fresh.tmp"
|
||||
p.write_text("x")
|
||||
dg.track(str(p), "temp", silent=True)
|
||||
summary = dg.quick()
|
||||
assert summary["deleted"] == 0
|
||||
assert p.exists()
|
||||
|
||||
def test_quick_preserves_protected_top_level_dirs(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
for d in ("logs", "memories", "sessions", "cron", "cache"):
|
||||
(_isolate_env / d).mkdir()
|
||||
dg.quick()
|
||||
for d in ("logs", "memories", "sessions", "cron", "cache"):
|
||||
assert (_isolate_env / d).exists(), f"{d}/ should be preserved"
|
||||
|
||||
|
||||
class TestStatus:
|
||||
def test_empty_status(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
s = dg.status()
|
||||
assert s["total_tracked"] == 0
|
||||
assert s["top10"] == []
|
||||
|
||||
def test_status_with_entries(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
p = _isolate_env / "big.tmp"
|
||||
p.write_text("y" * 100)
|
||||
dg.track(str(p), "temp", silent=True)
|
||||
s = dg.status()
|
||||
assert s["total_tracked"] == 1
|
||||
assert len(s["top10"]) == 1
|
||||
rendered = dg.format_status(s)
|
||||
assert "temp" in rendered
|
||||
assert "big.tmp" in rendered
|
||||
|
||||
|
||||
class TestDryRun:
|
||||
def test_classifies_by_category(self, _isolate_env):
|
||||
dg = _load_lib()
|
||||
test_f = _isolate_env / "test_x.py"
|
||||
test_f.write_text("x")
|
||||
big = _isolate_env / "big.bin"
|
||||
big.write_bytes(b"z" * 10)
|
||||
dg.track(str(test_f), "test", silent=True)
|
||||
dg.track(str(big), "other", silent=True)
|
||||
auto, prompt = dg.dry_run()
|
||||
# test → auto, other → neither (doesn't hit any rule)
|
||||
assert any(i["path"] == str(test_f) for i in auto)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin hooks tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPostToolCallHook:
|
||||
def test_write_file_test_pattern_tracked(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
p = _isolate_env / "test_created.py"
|
||||
p.write_text("x")
|
||||
pi._on_post_tool_call(
|
||||
tool_name="write_file",
|
||||
args={"path": str(p), "content": "x"},
|
||||
result="OK",
|
||||
task_id="t1", session_id="s1",
|
||||
)
|
||||
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
|
||||
data = json.loads(tracked_file.read_text())
|
||||
assert len(data) == 1
|
||||
assert data[0]["category"] == "test"
|
||||
|
||||
def test_write_file_non_test_not_tracked(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
p = _isolate_env / "notes.md"
|
||||
p.write_text("x")
|
||||
pi._on_post_tool_call(
|
||||
tool_name="write_file",
|
||||
args={"path": str(p), "content": "x"},
|
||||
result="OK",
|
||||
task_id="t2", session_id="s2",
|
||||
)
|
||||
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
|
||||
assert not tracked_file.exists() or tracked_file.read_text().strip() == "[]"
|
||||
|
||||
def test_terminal_command_picks_up_paths(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
p = _isolate_env / "tmp_created.log"
|
||||
p.write_text("x")
|
||||
pi._on_post_tool_call(
|
||||
tool_name="terminal",
|
||||
args={"command": f"touch {p}"},
|
||||
result=f"created {p}\n",
|
||||
task_id="t3", session_id="s3",
|
||||
)
|
||||
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
|
||||
data = json.loads(tracked_file.read_text())
|
||||
assert any(Path(i["path"]) == p.resolve() for i in data)
|
||||
|
||||
def test_ignores_unrelated_tool(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
pi._on_post_tool_call(
|
||||
tool_name="read_file",
|
||||
args={"path": str(_isolate_env / "test_x.py")},
|
||||
result="contents",
|
||||
task_id="t4", session_id="s4",
|
||||
)
|
||||
# read_file should never trigger tracking.
|
||||
tracked_file = _isolate_env / "disk-cleanup" / "tracked.json"
|
||||
assert not tracked_file.exists() or tracked_file.read_text().strip() == "[]"
|
||||
|
||||
|
||||
class TestOnSessionEndHook:
|
||||
def test_runs_quick_when_test_files_tracked(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
p = _isolate_env / "test_cleanup.py"
|
||||
p.write_text("x")
|
||||
pi._on_post_tool_call(
|
||||
tool_name="write_file",
|
||||
args={"path": str(p), "content": "x"},
|
||||
result="OK",
|
||||
task_id="", session_id="s1",
|
||||
)
|
||||
assert p.exists()
|
||||
pi._on_session_end(session_id="s1", completed=True, interrupted=False)
|
||||
assert not p.exists(), "test file should be auto-deleted"
|
||||
|
||||
def test_noop_when_no_test_tracked(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
# Nothing tracked → on_session_end should not raise.
|
||||
pi._on_session_end(session_id="empty", completed=True, interrupted=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Slash command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSlashCommand:
|
||||
def test_help(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
out = pi._handle_slash("help")
|
||||
assert "disk-cleanup" in out
|
||||
assert "status" in out
|
||||
|
||||
def test_status_empty(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
out = pi._handle_slash("status")
|
||||
assert "nothing tracked" in out
|
||||
|
||||
def test_track_rejects_missing(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
out = pi._handle_slash(
|
||||
f"track {_isolate_env / 'nope.txt'} temp"
|
||||
)
|
||||
assert "Not tracked" in out
|
||||
|
||||
def test_track_rejects_bad_category(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
p = _isolate_env / "a.tmp"
|
||||
p.write_text("x")
|
||||
out = pi._handle_slash(f"track {p} banana")
|
||||
assert "Unknown category" in out
|
||||
|
||||
def test_track_and_forget(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
p = _isolate_env / "a.tmp"
|
||||
p.write_text("x")
|
||||
out = pi._handle_slash(f"track {p} temp")
|
||||
assert "Tracked" in out
|
||||
out = pi._handle_slash(f"forget {p}")
|
||||
assert "Removed 1" in out
|
||||
|
||||
def test_unknown_subcommand(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
out = pi._handle_slash("foobar")
|
||||
assert "Unknown subcommand" in out
|
||||
|
||||
def test_quick_on_empty(self, _isolate_env):
|
||||
pi = _load_plugin_init()
|
||||
out = pi._handle_slash("quick")
|
||||
assert "Cleaned 0 files" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bundled-plugin discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBundledDiscovery:
|
||||
def _write_enabled_config(self, hermes_home, names):
|
||||
"""Write plugins.enabled allow-list to config.yaml."""
|
||||
import yaml
|
||||
cfg_path = hermes_home / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({"plugins": {"enabled": list(names)}}))
|
||||
|
||||
def test_disk_cleanup_discovered_but_not_loaded_by_default(self, _isolate_env):
|
||||
"""Bundled plugins are discovered but NOT loaded without opt-in."""
|
||||
from hermes_cli import plugins as pmod
|
||||
mgr = pmod.PluginManager()
|
||||
mgr.discover_and_load()
|
||||
# Discovered — appears in the registry
|
||||
assert "disk-cleanup" in mgr._plugins
|
||||
loaded = mgr._plugins["disk-cleanup"]
|
||||
assert loaded.manifest.source == "bundled"
|
||||
# But NOT enabled — no hooks or commands registered
|
||||
assert not loaded.enabled
|
||||
assert loaded.error and "not enabled" in loaded.error
|
||||
|
||||
def test_disk_cleanup_loads_when_enabled(self, _isolate_env):
|
||||
"""Adding to plugins.enabled activates the bundled plugin."""
|
||||
self._write_enabled_config(_isolate_env, ["disk-cleanup"])
|
||||
from hermes_cli import plugins as pmod
|
||||
mgr = pmod.PluginManager()
|
||||
mgr.discover_and_load()
|
||||
loaded = mgr._plugins["disk-cleanup"]
|
||||
assert loaded.enabled
|
||||
assert "post_tool_call" in loaded.hooks_registered
|
||||
assert "on_session_end" in loaded.hooks_registered
|
||||
assert "disk-cleanup" in loaded.commands_registered
|
||||
|
||||
def test_disabled_beats_enabled(self, _isolate_env):
|
||||
"""plugins.disabled wins even if the plugin is also in plugins.enabled."""
|
||||
import yaml
|
||||
cfg_path = _isolate_env / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"plugins": {
|
||||
"enabled": ["disk-cleanup"],
|
||||
"disabled": ["disk-cleanup"],
|
||||
}
|
||||
}))
|
||||
from hermes_cli import plugins as pmod
|
||||
mgr = pmod.PluginManager()
|
||||
mgr.discover_and_load()
|
||||
loaded = mgr._plugins["disk-cleanup"]
|
||||
assert not loaded.enabled
|
||||
assert loaded.error == "disabled via config"
|
||||
|
||||
def test_memory_and_context_engine_subdirs_skipped(self, _isolate_env):
|
||||
"""Bundled scan must NOT pick up plugins/memory or plugins/context_engine
|
||||
as top-level plugins — they have their own discovery paths."""
|
||||
self._write_enabled_config(
|
||||
_isolate_env, ["memory", "context_engine", "disk-cleanup"]
|
||||
)
|
||||
from hermes_cli import plugins as pmod
|
||||
mgr = pmod.PluginManager()
|
||||
mgr.discover_and_load()
|
||||
assert "memory" not in mgr._plugins
|
||||
assert "context_engine" not in mgr._plugins
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Tests for plugins.google_meet.audio_bridge (v2).
|
||||
|
||||
Covers the platform gating and pactl / system_profiler plumbing
|
||||
without actually invoking those tools on the host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_home(tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
yield hermes_home
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Linux setup / teardown
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _linux_pactl_result(stdout: str) -> MagicMock:
|
||||
"""Build a fake CompletedProcess-ish object for subprocess.run."""
|
||||
m = MagicMock()
|
||||
m.stdout = stdout
|
||||
m.stderr = ""
|
||||
m.returncode = 0
|
||||
return m
|
||||
|
||||
|
||||
def test_setup_linux_loads_null_sink_and_virtual_source():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def _fake_run(argv, **kwargs):
|
||||
calls.append(list(argv))
|
||||
# First call = null-sink → module id 42
|
||||
# Second call = virtual-source → module id 43
|
||||
if "module-null-sink" in argv:
|
||||
return _linux_pactl_result("42\n")
|
||||
if "module-virtual-source" in argv:
|
||||
return _linux_pactl_result("43\n")
|
||||
raise AssertionError(f"unexpected pactl invocation: {argv}")
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Linux"), \
|
||||
patch("plugins.google_meet.audio_bridge.subprocess.run",
|
||||
side_effect=_fake_run):
|
||||
br = AudioBridge()
|
||||
info = br.setup()
|
||||
|
||||
# Two pactl load-module calls, in order.
|
||||
assert len(calls) == 2
|
||||
assert calls[0][0] == "pactl" and calls[0][1] == "load-module"
|
||||
assert "module-null-sink" in calls[0]
|
||||
assert any(a.startswith("sink_name=hermes_meet_sink") for a in calls[0])
|
||||
assert calls[1][0] == "pactl" and calls[1][1] == "load-module"
|
||||
assert "module-virtual-source" in calls[1]
|
||||
assert any(a.startswith("source_name=hermes_meet_src") for a in calls[1])
|
||||
assert any("master=hermes_meet_sink.monitor" in a for a in calls[1])
|
||||
|
||||
# Dict shape.
|
||||
assert info["platform"] == "linux"
|
||||
assert info["device_name"] == "hermes_meet_src"
|
||||
assert info["write_target"] == "hermes_meet_sink"
|
||||
assert info["sample_rate"] == 48000
|
||||
assert info["channels"] == 2
|
||||
assert info["module_ids"] == [42, 43]
|
||||
|
||||
# Properties.
|
||||
assert br.device_name == "hermes_meet_src"
|
||||
assert br.write_target == "hermes_meet_sink"
|
||||
|
||||
|
||||
def test_teardown_linux_unloads_modules_in_reverse_order():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
def _setup_run(argv, **kwargs):
|
||||
if "module-null-sink" in argv:
|
||||
return _linux_pactl_result("42\n")
|
||||
return _linux_pactl_result("43\n")
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Linux"), \
|
||||
patch("plugins.google_meet.audio_bridge.subprocess.run",
|
||||
side_effect=_setup_run):
|
||||
br = AudioBridge()
|
||||
br.setup()
|
||||
|
||||
unload_calls: list[list[str]] = []
|
||||
|
||||
def _teardown_run(argv, **kwargs):
|
||||
unload_calls.append(list(argv))
|
||||
return _linux_pactl_result("")
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.subprocess.run",
|
||||
side_effect=_teardown_run):
|
||||
br.teardown()
|
||||
|
||||
# Two unload calls, in reverse order: 43 (virtual-source) then 42 (sink).
|
||||
assert [c[1] for c in unload_calls] == ["unload-module", "unload-module"]
|
||||
assert unload_calls[0][2] == "43"
|
||||
assert unload_calls[1][2] == "42"
|
||||
|
||||
# Second teardown is a no-op.
|
||||
with patch("plugins.google_meet.audio_bridge.subprocess.run") as run_mock:
|
||||
br.teardown()
|
||||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
def test_setup_linux_parses_module_id_from_multi_line_output():
|
||||
"""Some pactl builds include trailing whitespace / notices."""
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
def _fake_run(argv, **kwargs):
|
||||
if "module-null-sink" in argv:
|
||||
return _linux_pactl_result("42 \n")
|
||||
return _linux_pactl_result("43\n")
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Linux"), \
|
||||
patch("plugins.google_meet.audio_bridge.subprocess.run",
|
||||
side_effect=_fake_run):
|
||||
br = AudioBridge()
|
||||
info = br.setup()
|
||||
|
||||
assert info["module_ids"] == [42, 43]
|
||||
|
||||
|
||||
def test_setup_linux_pactl_missing_raises_clean_error():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Linux"), \
|
||||
patch("plugins.google_meet.audio_bridge.subprocess.run",
|
||||
side_effect=FileNotFoundError("pactl")):
|
||||
br = AudioBridge()
|
||||
with pytest.raises(RuntimeError, match="pactl"):
|
||||
br.setup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# macOS setup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BH_PRESENT = (
|
||||
"Audio:\n"
|
||||
" Devices:\n"
|
||||
" BlackHole 2ch:\n"
|
||||
" Manufacturer: Existential Audio\n"
|
||||
)
|
||||
|
||||
_BH_ABSENT = (
|
||||
"Audio:\n"
|
||||
" Devices:\n"
|
||||
" MacBook Pro Microphone:\n"
|
||||
" Default Input: Yes\n"
|
||||
)
|
||||
|
||||
|
||||
def test_setup_darwin_returns_blackhole_when_present():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Darwin"), \
|
||||
patch("plugins.google_meet.audio_bridge.subprocess.check_output",
|
||||
return_value=_BH_PRESENT) as check:
|
||||
br = AudioBridge()
|
||||
info = br.setup()
|
||||
|
||||
check.assert_called_once()
|
||||
argv = check.call_args.args[0]
|
||||
assert argv[0] == "system_profiler"
|
||||
assert "SPAudioDataType" in argv
|
||||
|
||||
assert info["platform"] == "darwin"
|
||||
assert info["device_name"] == "BlackHole 2ch"
|
||||
assert info["write_target"] == "BlackHole 2ch"
|
||||
assert info["module_ids"] == []
|
||||
assert info["sample_rate"] == 48000
|
||||
assert info["channels"] == 2
|
||||
|
||||
# teardown is a no-op on darwin (no modules to unload).
|
||||
with patch("plugins.google_meet.audio_bridge.subprocess.run") as run_mock:
|
||||
br.teardown()
|
||||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
def test_setup_darwin_raises_when_blackhole_missing():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Darwin"), \
|
||||
patch("plugins.google_meet.audio_bridge.subprocess.check_output",
|
||||
return_value=_BH_ABSENT):
|
||||
br = AudioBridge()
|
||||
with pytest.raises(RuntimeError, match="BlackHole"):
|
||||
br.setup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Windows / unsupported
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_setup_windows_raises():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Windows"):
|
||||
br = AudioBridge()
|
||||
with pytest.raises(RuntimeError, match="not supported"):
|
||||
br.setup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# chrome_fake_audio_flags
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_chrome_fake_audio_flags_linux():
|
||||
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Linux"):
|
||||
flags = chrome_fake_audio_flags(
|
||||
{"platform": "linux", "device_name": "hermes_meet_src"}
|
||||
)
|
||||
assert "--use-fake-ui-for-media-stream" in flags
|
||||
|
||||
|
||||
def test_chrome_fake_audio_flags_darwin():
|
||||
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Darwin"):
|
||||
flags = chrome_fake_audio_flags(
|
||||
{"platform": "darwin", "device_name": "BlackHole 2ch"}
|
||||
)
|
||||
assert "--use-fake-ui-for-media-stream" in flags
|
||||
|
||||
|
||||
def test_chrome_fake_audio_flags_windows_raises():
|
||||
from plugins.google_meet.audio_bridge import chrome_fake_audio_flags
|
||||
|
||||
with patch("plugins.google_meet.audio_bridge.platform.system",
|
||||
return_value="Windows"):
|
||||
with pytest.raises(RuntimeError):
|
||||
chrome_fake_audio_flags({"platform": "windows"})
|
||||
|
||||
|
||||
def test_property_access_before_setup_raises():
|
||||
from plugins.google_meet.audio_bridge import AudioBridge
|
||||
|
||||
br = AudioBridge()
|
||||
with pytest.raises(RuntimeError):
|
||||
_ = br.device_name
|
||||
with pytest.raises(RuntimeError):
|
||||
_ = br.write_target
|
||||
@@ -0,0 +1,674 @@
|
||||
"""Tests for the google_meet node primitive.
|
||||
|
||||
Covers protocol helpers, the file-backed registry, the server's
|
||||
token-and-dispatch machinery, a mocked client, and the CLI plumbing.
|
||||
We never open a real socket — websockets.serve / websockets.sync.client
|
||||
are fully mocked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_home(tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
yield hermes_home
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# protocol.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_protocol_encode_decode_roundtrip():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
msg = protocol.make_request("ping", "tok", {"x": 1}, req_id="abc")
|
||||
raw = protocol.encode(msg)
|
||||
out = protocol.decode(raw)
|
||||
assert out == msg
|
||||
assert out["type"] == "ping"
|
||||
assert out["id"] == "abc"
|
||||
assert out["token"] == "tok"
|
||||
assert out["payload"] == {"x": 1}
|
||||
|
||||
|
||||
def test_protocol_make_request_autogenerates_id():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
a = protocol.make_request("ping", "tok", {})
|
||||
b = protocol.make_request("ping", "tok", {})
|
||||
assert a["id"] != b["id"]
|
||||
assert len(a["id"]) >= 16 # uuid4 hex
|
||||
|
||||
|
||||
def test_protocol_make_request_rejects_bad_input():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
protocol.make_request("", "tok", {})
|
||||
with pytest.raises(ValueError):
|
||||
protocol.make_request("unknown_type", "tok", {})
|
||||
with pytest.raises(ValueError):
|
||||
protocol.make_request("ping", "tok", "not a dict") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_protocol_decode_raises_on_malformed():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
protocol.decode("not json at all")
|
||||
with pytest.raises(ValueError):
|
||||
protocol.decode("[]") # list, not object
|
||||
with pytest.raises(ValueError):
|
||||
protocol.decode(json.dumps({"id": "x"})) # missing type
|
||||
with pytest.raises(ValueError):
|
||||
protocol.decode(json.dumps({"type": "ping"})) # missing id
|
||||
|
||||
|
||||
def test_protocol_validate_request_happy_path():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
msg = protocol.make_request("status", "secret", {})
|
||||
ok, reason = protocol.validate_request(msg, "secret")
|
||||
assert ok is True
|
||||
assert reason == ""
|
||||
|
||||
|
||||
def test_protocol_validate_request_rejects_bad_token():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
msg = protocol.make_request("status", "wrong", {})
|
||||
ok, reason = protocol.validate_request(msg, "right")
|
||||
assert ok is False
|
||||
assert "token" in reason.lower()
|
||||
|
||||
|
||||
def test_protocol_validate_request_rejects_unknown_type():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
raw = {"type": "nope", "id": "1", "token": "t", "payload": {}}
|
||||
ok, reason = protocol.validate_request(raw, "t")
|
||||
assert ok is False
|
||||
assert "unknown" in reason.lower()
|
||||
|
||||
|
||||
def test_protocol_validate_request_rejects_missing_id():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
raw = {"type": "ping", "token": "t", "payload": {}}
|
||||
ok, reason = protocol.validate_request(raw, "t")
|
||||
assert ok is False
|
||||
assert "id" in reason.lower()
|
||||
|
||||
|
||||
def test_protocol_validate_request_rejects_non_dict_payload():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
raw = {"type": "ping", "id": "1", "token": "t", "payload": "oops"}
|
||||
ok, reason = protocol.validate_request(raw, "t")
|
||||
assert ok is False
|
||||
|
||||
|
||||
def test_protocol_error_envelope_shape():
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
err = protocol.make_error("abc", "nope")
|
||||
assert err == {"type": "error", "id": "abc", "error": "nope"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# registry.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_registry_add_get_roundtrip_persists(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
p = tmp_path / "nodes.json"
|
||||
r = NodeRegistry(path=p)
|
||||
r.add("mac", "ws://mac.local:18789", "deadbeef")
|
||||
|
||||
# Second instance sees it.
|
||||
r2 = NodeRegistry(path=p)
|
||||
entry = r2.get("mac")
|
||||
assert entry is not None
|
||||
assert entry["name"] == "mac"
|
||||
assert entry["url"] == "ws://mac.local:18789"
|
||||
assert entry["token"] == "deadbeef"
|
||||
assert "added_at" in entry
|
||||
|
||||
|
||||
def test_registry_get_returns_none_when_missing(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
assert r.get("ghost") is None
|
||||
|
||||
|
||||
def test_registry_remove(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
r.add("a", "ws://a", "t")
|
||||
assert r.remove("a") is True
|
||||
assert r.get("a") is None
|
||||
assert r.remove("a") is False # idempotent
|
||||
|
||||
|
||||
def test_registry_list_all_sorted(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
r.add("zeta", "ws://z", "t1")
|
||||
r.add("alpha", "ws://a", "t2")
|
||||
names = [n["name"] for n in r.list_all()]
|
||||
assert names == ["alpha", "zeta"]
|
||||
|
||||
|
||||
def test_registry_resolve_auto_picks_single(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
r.add("mac", "ws://mac", "t")
|
||||
picked = r.resolve(None)
|
||||
assert picked is not None
|
||||
assert picked["name"] == "mac"
|
||||
|
||||
|
||||
def test_registry_resolve_ambiguous_returns_none(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
r.add("a", "ws://a", "t")
|
||||
r.add("b", "ws://b", "t")
|
||||
assert r.resolve(None) is None
|
||||
|
||||
|
||||
def test_registry_resolve_empty_returns_none(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
assert r.resolve(None) is None
|
||||
|
||||
|
||||
def test_registry_resolve_by_name(tmp_path):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
r = NodeRegistry(path=tmp_path / "n.json")
|
||||
r.add("a", "ws://a", "t")
|
||||
r.add("b", "ws://b", "t")
|
||||
picked = r.resolve("b")
|
||||
assert picked is not None
|
||||
assert picked["name"] == "b"
|
||||
assert r.resolve("ghost") is None
|
||||
|
||||
|
||||
def test_registry_defaults_to_hermes_home(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
# _isolate_home already set HERMES_HOME to tmp_path/.hermes; the
|
||||
# registry default path must live inside that tree.
|
||||
r = NodeRegistry()
|
||||
r.add("x", "ws://x", "t")
|
||||
expected = Path(tmp_path) / ".hermes" / "workspace" / "meetings" / "nodes.json"
|
||||
assert expected.is_file()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# server.py — token + dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_server_ensure_token_generates_and_persists(tmp_path):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
|
||||
p = tmp_path / "tok.json"
|
||||
s1 = NodeServer(token_path=p)
|
||||
t1 = s1.ensure_token()
|
||||
assert isinstance(t1, str) and len(t1) == 32
|
||||
|
||||
# Reuse on a fresh instance.
|
||||
s2 = NodeServer(token_path=p)
|
||||
t2 = s2.ensure_token()
|
||||
assert t1 == t2
|
||||
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
assert data["token"] == t1
|
||||
assert "generated_at" in data
|
||||
|
||||
|
||||
def test_server_get_token_is_idempotent(tmp_path):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
assert s.get_token() == s.get_token()
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro) if False else asyncio.run(coro)
|
||||
|
||||
|
||||
def test_server_handle_request_rejects_bad_token(tmp_path):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
s.ensure_token()
|
||||
bad = protocol.make_request("ping", "not-the-token", {})
|
||||
resp = asyncio.run(s._handle_request(bad))
|
||||
assert resp["type"] == "error"
|
||||
assert "token" in resp["error"].lower()
|
||||
|
||||
|
||||
def test_server_handle_request_ping(tmp_path):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json", display_name="node-x")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("ping", tok, {})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "pong"
|
||||
assert resp["id"] == req["id"]
|
||||
assert resp["payload"]["display_name"] == "node-x"
|
||||
|
||||
|
||||
def test_server_handle_request_status_dispatches_to_pm(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
monkeypatch.setattr(pm, "status",
|
||||
lambda: {"ok": True, "alive": True, "meetingId": "abc"})
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("status", tok, {})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "response"
|
||||
assert resp["id"] == req["id"]
|
||||
assert resp["payload"] == {"ok": True, "alive": True, "meetingId": "abc"}
|
||||
|
||||
|
||||
def test_server_handle_request_start_bot_dispatches(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_start(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"ok": True, "pid": 42, "meeting_id": "abc-defg-hij"}
|
||||
|
||||
monkeypatch.setattr(pm, "start", fake_start)
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("start_bot", tok, {
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"guest_name": "Bot",
|
||||
"duration": "30m",
|
||||
})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "response"
|
||||
assert resp["payload"]["ok"] is True
|
||||
assert captured["url"] == "https://meet.google.com/abc-defg-hij"
|
||||
assert captured["guest_name"] == "Bot"
|
||||
assert captured["duration"] == "30m"
|
||||
|
||||
|
||||
def test_server_handle_request_start_bot_missing_url(tmp_path):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("start_bot", tok, {"guest_name": "x"})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "error"
|
||||
assert "url" in resp["error"]
|
||||
|
||||
|
||||
def test_server_handle_request_stop_dispatches(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
got = {}
|
||||
|
||||
def fake_stop(*, reason="requested"):
|
||||
got["reason"] = reason
|
||||
return {"ok": True, "reason": reason}
|
||||
|
||||
monkeypatch.setattr(pm, "stop", fake_stop)
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("stop", tok, {"reason": "user-cancel"})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "response"
|
||||
assert got["reason"] == "user-cancel"
|
||||
|
||||
|
||||
def test_server_handle_request_transcript(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
got = {}
|
||||
|
||||
def fake_transcript(last=None):
|
||||
got["last"] = last
|
||||
return {"ok": True, "lines": ["a", "b"], "total": 2}
|
||||
|
||||
monkeypatch.setattr(pm, "transcript", fake_transcript)
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("transcript", tok, {"last": 5})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "response"
|
||||
assert resp["payload"]["lines"] == ["a", "b"]
|
||||
assert got["last"] == 5
|
||||
|
||||
|
||||
def test_server_handle_request_say_enqueues_when_active(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
out = tmp_path / "meet-out"
|
||||
out.mkdir()
|
||||
monkeypatch.setattr(pm, "_read_active",
|
||||
lambda: {"pid": 1, "meeting_id": "m", "out_dir": str(out)})
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("say", tok, {"text": "hello"})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "response"
|
||||
assert resp["payload"]["ok"] is True
|
||||
assert resp["payload"]["enqueued"] is True
|
||||
q = (out / "say_queue.jsonl").read_text(encoding="utf-8").strip().splitlines()
|
||||
assert len(q) == 1
|
||||
assert json.loads(q[0])["text"] == "hello"
|
||||
|
||||
|
||||
def test_server_handle_request_say_without_active_still_ok(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
monkeypatch.setattr(pm, "_read_active", lambda: None)
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("say", tok, {"text": "hi"})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "response"
|
||||
assert resp["payload"]["ok"] is True
|
||||
assert resp["payload"]["enqueued"] is False
|
||||
|
||||
|
||||
def test_server_handle_request_wraps_pm_exceptions(tmp_path, monkeypatch):
|
||||
from plugins.google_meet.node.server import NodeServer
|
||||
from plugins.google_meet.node import protocol
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
def boom():
|
||||
raise ValueError("kaboom")
|
||||
|
||||
monkeypatch.setattr(pm, "status", boom)
|
||||
|
||||
s = NodeServer(token_path=tmp_path / "t.json")
|
||||
tok = s.ensure_token()
|
||||
req = protocol.make_request("status", tok, {})
|
||||
resp = asyncio.run(s._handle_request(req))
|
||||
assert resp["type"] == "error"
|
||||
assert "kaboom" in resp["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# client.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _FakeWS:
|
||||
"""Minimal context-manager stand-in for websockets.sync.client.connect."""
|
||||
|
||||
def __init__(self, reply_builder):
|
||||
self._reply_builder = reply_builder
|
||||
self.sent = []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def send(self, raw):
|
||||
self.sent.append(raw)
|
||||
|
||||
def recv(self, timeout=None):
|
||||
return self._reply_builder(self.sent[-1])
|
||||
|
||||
|
||||
def _install_fake_ws(monkeypatch, reply_builder):
|
||||
fake_ws_holder = {}
|
||||
|
||||
def _connect(url, **kwargs):
|
||||
ws = _FakeWS(reply_builder)
|
||||
fake_ws_holder["ws"] = ws
|
||||
fake_ws_holder["url"] = url
|
||||
fake_ws_holder["kwargs"] = kwargs
|
||||
return ws
|
||||
|
||||
# Patch the concrete import site inside client._rpc
|
||||
import websockets.sync.client as wsc # type: ignore
|
||||
monkeypatch.setattr(wsc, "connect", _connect)
|
||||
return fake_ws_holder
|
||||
|
||||
|
||||
def test_client_rpc_sends_correct_envelope_and_parses_response(monkeypatch):
|
||||
from plugins.google_meet.node.client import NodeClient
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
def reply(raw_out):
|
||||
req = protocol.decode(raw_out)
|
||||
return protocol.encode(protocol.make_response(req["id"], {"ok": True, "echo": req["type"]}))
|
||||
|
||||
holder = _install_fake_ws(monkeypatch, reply)
|
||||
|
||||
c = NodeClient("ws://remote:1", "tok123")
|
||||
out = c._rpc("ping", {"hello": 1})
|
||||
assert out == {"ok": True, "echo": "ping"}
|
||||
|
||||
sent = json.loads(holder["ws"].sent[0])
|
||||
assert sent["type"] == "ping"
|
||||
assert sent["token"] == "tok123"
|
||||
assert sent["payload"] == {"hello": 1}
|
||||
assert sent["id"] # non-empty
|
||||
assert holder["url"] == "ws://remote:1"
|
||||
|
||||
|
||||
def test_client_rpc_raises_on_error_envelope(monkeypatch):
|
||||
from plugins.google_meet.node.client import NodeClient
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
def reply(raw_out):
|
||||
req = protocol.decode(raw_out)
|
||||
return protocol.encode(protocol.make_error(req["id"], "nope"))
|
||||
|
||||
_install_fake_ws(monkeypatch, reply)
|
||||
|
||||
c = NodeClient("ws://x", "t")
|
||||
with pytest.raises(RuntimeError, match="nope"):
|
||||
c._rpc("ping", {})
|
||||
|
||||
|
||||
def test_client_rpc_raises_on_id_mismatch(monkeypatch):
|
||||
from plugins.google_meet.node.client import NodeClient
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
def reply(raw_out):
|
||||
return protocol.encode(protocol.make_response("different-id", {"ok": True}))
|
||||
|
||||
_install_fake_ws(monkeypatch, reply)
|
||||
|
||||
c = NodeClient("ws://x", "t")
|
||||
with pytest.raises(RuntimeError, match="mismatch"):
|
||||
c._rpc("ping", {})
|
||||
|
||||
|
||||
def test_client_convenience_methods_hit_correct_types(monkeypatch):
|
||||
from plugins.google_meet.node.client import NodeClient
|
||||
from plugins.google_meet.node import protocol
|
||||
|
||||
seen = []
|
||||
|
||||
def reply(raw_out):
|
||||
req = protocol.decode(raw_out)
|
||||
seen.append((req["type"], req["payload"]))
|
||||
return protocol.encode(protocol.make_response(req["id"], {"ok": True}))
|
||||
|
||||
_install_fake_ws(monkeypatch, reply)
|
||||
|
||||
c = NodeClient("ws://x", "t")
|
||||
c.start_bot("https://meet.google.com/a-b-c", guest_name="G", duration="10m")
|
||||
c.stop()
|
||||
c.status()
|
||||
c.transcript(last=3)
|
||||
c.say("hi")
|
||||
c.ping()
|
||||
|
||||
types = [t for t, _ in seen]
|
||||
assert types == ["start_bot", "stop", "status", "transcript", "say", "ping"]
|
||||
# Check specific payload routing
|
||||
assert seen[0][1]["url"] == "https://meet.google.com/a-b-c"
|
||||
assert seen[0][1]["guest_name"] == "G"
|
||||
assert seen[0][1]["duration"] == "10m"
|
||||
assert seen[3][1]["last"] == 3
|
||||
assert seen[4][1]["text"] == "hi"
|
||||
|
||||
|
||||
def test_client_init_rejects_bad_args():
|
||||
from plugins.google_meet.node.client import NodeClient
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
NodeClient("", "t")
|
||||
with pytest.raises(ValueError):
|
||||
NodeClient("ws://x", "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cli.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_parser():
|
||||
from plugins.google_meet.node.cli import register_cli
|
||||
|
||||
parser = argparse.ArgumentParser(prog="meet-node-test")
|
||||
register_cli(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def test_cli_approve_list_remove(capsys):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
p = _build_parser()
|
||||
|
||||
args = p.parse_args(["approve", "mac", "ws://mac:1", "tok"])
|
||||
rc = args.func(args)
|
||||
assert rc == 0
|
||||
assert NodeRegistry().get("mac") is not None
|
||||
|
||||
args = p.parse_args(["list"])
|
||||
rc = args.func(args)
|
||||
assert rc == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "mac" in out
|
||||
assert "ws://mac:1" in out
|
||||
|
||||
args = p.parse_args(["remove", "mac"])
|
||||
rc = args.func(args)
|
||||
assert rc == 0
|
||||
assert NodeRegistry().get("mac") is None
|
||||
|
||||
|
||||
def test_cli_list_empty(capsys):
|
||||
p = _build_parser()
|
||||
args = p.parse_args(["list"])
|
||||
rc = args.func(args)
|
||||
assert rc == 0
|
||||
assert "no nodes" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_cli_remove_missing_returns_nonzero():
|
||||
p = _build_parser()
|
||||
args = p.parse_args(["remove", "ghost"])
|
||||
rc = args.func(args)
|
||||
assert rc == 1
|
||||
|
||||
|
||||
def test_cli_status_pings_via_node_client(capsys, monkeypatch):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
from plugins.google_meet.node import cli as node_cli
|
||||
|
||||
NodeRegistry().add("mac", "ws://mac:1", "tok")
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, url, token):
|
||||
assert url == "ws://mac:1"
|
||||
assert token == "tok"
|
||||
|
||||
def ping(self):
|
||||
return {"type": "pong", "display_name": "hermes-meet-node"}
|
||||
|
||||
monkeypatch.setattr(node_cli, "NodeClient", _FakeClient)
|
||||
|
||||
p = _build_parser()
|
||||
args = p.parse_args(["status", "mac"])
|
||||
rc = args.func(args)
|
||||
assert rc == 0
|
||||
out = capsys.readouterr().out.strip()
|
||||
data = json.loads(out)
|
||||
assert data["ok"] is True
|
||||
assert data["node"] == "mac"
|
||||
|
||||
|
||||
def test_cli_status_unknown_node_fails(capsys):
|
||||
p = _build_parser()
|
||||
args = p.parse_args(["status", "ghost"])
|
||||
rc = args.func(args)
|
||||
assert rc == 1
|
||||
|
||||
|
||||
def test_cli_status_reports_client_error(capsys, monkeypatch):
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
from plugins.google_meet.node import cli as node_cli
|
||||
|
||||
NodeRegistry().add("mac", "ws://mac:1", "tok")
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, url, token):
|
||||
pass
|
||||
|
||||
def ping(self):
|
||||
raise RuntimeError("connection refused")
|
||||
|
||||
monkeypatch.setattr(node_cli, "NodeClient", _FakeClient)
|
||||
|
||||
p = _build_parser()
|
||||
args = p.parse_args(["status", "mac"])
|
||||
rc = args.func(args)
|
||||
assert rc == 1
|
||||
data = json.loads(capsys.readouterr().out.strip())
|
||||
assert data["ok"] is False
|
||||
assert "connection refused" in data["error"]
|
||||
@@ -0,0 +1,813 @@
|
||||
"""Tests for the google_meet plugin.
|
||||
|
||||
Covers the safety-gated pieces that don't require Playwright:
|
||||
|
||||
* URL regex — only ``https://meet.google.com/`` URLs pass
|
||||
* Meeting-id extraction from Meet URLs
|
||||
* Status / transcript writes round-trip through the file-backed state
|
||||
* Tool handlers return well-formed JSON under all branches
|
||||
* Process manager refuses unsafe URLs and clears stale state cleanly
|
||||
* ``_on_session_end`` hook is defensive (no-ops when no bot active)
|
||||
|
||||
Does NOT spawn a real Chromium — we mock ``subprocess.Popen`` where needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_home(tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
yield hermes_home
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# URL safety gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_is_safe_meet_url_accepts_standard_meet_codes():
|
||||
from plugins.google_meet.meet_bot import _is_safe_meet_url
|
||||
|
||||
assert _is_safe_meet_url("https://meet.google.com/abc-defg-hij")
|
||||
assert _is_safe_meet_url("https://meet.google.com/abc-defg-hij?pli=1")
|
||||
assert _is_safe_meet_url("https://meet.google.com/new")
|
||||
assert _is_safe_meet_url("https://meet.google.com/lookup/ABC123")
|
||||
|
||||
|
||||
def test_is_safe_meet_url_rejects_non_meet_urls():
|
||||
from plugins.google_meet.meet_bot import _is_safe_meet_url
|
||||
|
||||
# wrong host
|
||||
assert not _is_safe_meet_url("https://evil.example.com/abc-defg-hij")
|
||||
# wrong scheme
|
||||
assert not _is_safe_meet_url("http://meet.google.com/abc-defg-hij")
|
||||
# malformed code
|
||||
assert not _is_safe_meet_url("https://meet.google.com/not-a-meet-code")
|
||||
# subdomain hijack attempts
|
||||
assert not _is_safe_meet_url("https://meet.google.com.evil.com/abc-defg-hij")
|
||||
assert not _is_safe_meet_url("https://notmeet.google.com/abc-defg-hij")
|
||||
# empty / wrong type
|
||||
assert not _is_safe_meet_url("")
|
||||
assert not _is_safe_meet_url(None) # type: ignore[arg-type]
|
||||
assert not _is_safe_meet_url(123) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_meeting_id_extraction():
|
||||
from plugins.google_meet.meet_bot import _meeting_id_from_url
|
||||
|
||||
assert _meeting_id_from_url("https://meet.google.com/abc-defg-hij") == "abc-defg-hij"
|
||||
assert _meeting_id_from_url("https://meet.google.com/abc-defg-hij?pli=1") == "abc-defg-hij"
|
||||
# fallback for codes we can't parse (e.g. /new before redirect)
|
||||
fallback = _meeting_id_from_url("https://meet.google.com/new")
|
||||
assert fallback.startswith("meet-")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _BotState — transcript + status file round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_bot_state_dedupes_captions_and_flushes_status(tmp_path):
|
||||
from plugins.google_meet.meet_bot import _BotState
|
||||
|
||||
out = tmp_path / "session"
|
||||
state = _BotState(out_dir=out, meeting_id="abc-defg-hij",
|
||||
url="https://meet.google.com/abc-defg-hij")
|
||||
|
||||
state.record_caption("Alice", "Hey everyone")
|
||||
state.record_caption("Alice", "Hey everyone") # dup — ignored
|
||||
state.record_caption("Bob", "Let's start")
|
||||
|
||||
transcript = (out / "transcript.txt").read_text()
|
||||
assert "Alice: Hey everyone" in transcript
|
||||
assert "Bob: Let's start" in transcript
|
||||
# dedup — Alice line appears exactly once
|
||||
assert transcript.count("Alice: Hey everyone") == 1
|
||||
|
||||
status = json.loads((out / "status.json").read_text())
|
||||
assert status["meetingId"] == "abc-defg-hij"
|
||||
assert status["transcriptLines"] == 2
|
||||
assert status["transcriptPath"].endswith("transcript.txt")
|
||||
|
||||
|
||||
def test_bot_state_ignores_blank_text(tmp_path):
|
||||
from plugins.google_meet.meet_bot import _BotState
|
||||
|
||||
state = _BotState(out_dir=tmp_path / "s", meeting_id="x-y-z",
|
||||
url="https://meet.google.com/x-y-z")
|
||||
state.record_caption("Alice", "")
|
||||
state.record_caption("Alice", " ")
|
||||
state.record_caption("", "text but no speaker")
|
||||
|
||||
status = json.loads((tmp_path / "s" / "status.json").read_text())
|
||||
assert status["transcriptLines"] == 1
|
||||
# blank-speaker falls back to "Unknown"
|
||||
assert "Unknown: text but no speaker" in (tmp_path / "s" / "transcript.txt").read_text()
|
||||
|
||||
|
||||
def test_parse_duration():
|
||||
from plugins.google_meet.meet_bot import _parse_duration
|
||||
|
||||
assert _parse_duration("30m") == 30 * 60
|
||||
assert _parse_duration("2h") == 2 * 3600
|
||||
assert _parse_duration("90s") == 90
|
||||
assert _parse_duration("90") == 90
|
||||
assert _parse_duration("") is None
|
||||
assert _parse_duration("bogus") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# process_manager — refuses unsafe URLs, manages active pointer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_start_refuses_unsafe_url():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
res = pm.start("https://evil.example.com/abc-defg-hij")
|
||||
assert res["ok"] is False
|
||||
assert "refusing" in res["error"]
|
||||
|
||||
|
||||
def test_status_reports_no_active_meeting():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
assert pm.status() == {"ok": False, "reason": "no active meeting"}
|
||||
assert pm.transcript() == {"ok": False, "reason": "no active meeting"}
|
||||
assert pm.stop() == {"ok": False, "reason": "no active meeting"}
|
||||
|
||||
|
||||
def test_start_spawns_subprocess_and_writes_active_pointer(tmp_path):
|
||||
"""Verify start() wires env vars correctly and records the pid."""
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
class _FakeProc:
|
||||
def __init__(self, pid):
|
||||
self.pid = pid
|
||||
|
||||
captured_env = {}
|
||||
captured_argv = []
|
||||
|
||||
def _fake_popen(argv, **kwargs):
|
||||
captured_argv.extend(argv)
|
||||
captured_env.update(kwargs.get("env") or {})
|
||||
return _FakeProc(99999)
|
||||
|
||||
with patch.object(pm.subprocess, "Popen", side_effect=_fake_popen):
|
||||
# Also prevent pid liveness probe from stomping on our real pids
|
||||
with patch.object(pm, "_pid_alive", return_value=False):
|
||||
res = pm.start(
|
||||
"https://meet.google.com/abc-defg-hij",
|
||||
guest_name="Test Bot",
|
||||
duration="15m",
|
||||
)
|
||||
|
||||
assert res["ok"] is True
|
||||
assert res["meeting_id"] == "abc-defg-hij"
|
||||
assert res["pid"] == 99999
|
||||
assert captured_env["HERMES_MEET_URL"] == "https://meet.google.com/abc-defg-hij"
|
||||
assert captured_env["HERMES_MEET_GUEST_NAME"] == "Test Bot"
|
||||
assert captured_env["HERMES_MEET_DURATION"] == "15m"
|
||||
# python -m plugins.google_meet.meet_bot
|
||||
assert any("plugins.google_meet.meet_bot" in a for a in captured_argv)
|
||||
|
||||
# .active.json points at the bot
|
||||
active = pm._read_active()
|
||||
assert active is not None
|
||||
assert active["pid"] == 99999
|
||||
assert active["meeting_id"] == "abc-defg-hij"
|
||||
|
||||
|
||||
def test_transcript_reads_last_n_lines(tmp_path):
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
meeting_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
|
||||
meeting_dir.mkdir(parents=True)
|
||||
(meeting_dir / "transcript.txt").write_text(
|
||||
"[10:00:00] Alice: one\n"
|
||||
"[10:00:01] Bob: two\n"
|
||||
"[10:00:02] Alice: three\n"
|
||||
)
|
||||
pm._write_active({
|
||||
"pid": 0, "meeting_id": "abc-defg-hij",
|
||||
"out_dir": str(meeting_dir),
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"started_at": 0,
|
||||
})
|
||||
|
||||
res = pm.transcript(last=2)
|
||||
assert res["ok"] is True
|
||||
assert res["total"] == 3
|
||||
assert len(res["lines"]) == 2
|
||||
assert res["lines"][-1].endswith("Alice: three")
|
||||
|
||||
|
||||
def test_stop_signals_process_and_clears_pointer(tmp_path):
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
pm._write_active({
|
||||
"pid": 11111, "meeting_id": "x-y-z",
|
||||
"out_dir": str(tmp_path / "x-y-z"),
|
||||
"url": "https://meet.google.com/x-y-z",
|
||||
"started_at": 0,
|
||||
})
|
||||
|
||||
alive_seq = iter([True, True, False]) # alive at first, gone after SIGTERM
|
||||
def _alive(pid):
|
||||
try:
|
||||
return next(alive_seq)
|
||||
except StopIteration:
|
||||
return False
|
||||
|
||||
sent = []
|
||||
def _kill(pid, sig):
|
||||
sent.append((pid, sig))
|
||||
|
||||
with patch.object(pm, "_pid_alive", side_effect=_alive), \
|
||||
patch.object(pm.os, "kill", side_effect=_kill), \
|
||||
patch.object(pm.time, "sleep", lambda _s: None):
|
||||
res = pm.stop()
|
||||
|
||||
assert res["ok"] is True
|
||||
assert (11111, signal.SIGTERM) in sent
|
||||
# .active.json cleared
|
||||
assert pm._read_active() is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool handlers — JSON shape + safety gates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_meet_join_handler_missing_url_returns_error():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
|
||||
out = json.loads(handle_meet_join({}))
|
||||
assert out["success"] is False
|
||||
assert "url is required" in out["error"]
|
||||
|
||||
|
||||
def test_meet_join_handler_respects_safety_gate():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
|
||||
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=True):
|
||||
out = json.loads(handle_meet_join({"url": "https://evil.example.com/foo"}))
|
||||
assert out["success"] is False
|
||||
assert "refusing" in out["error"]
|
||||
|
||||
|
||||
def test_meet_join_handler_returns_error_when_playwright_missing():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
|
||||
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=False):
|
||||
out = json.loads(handle_meet_join({"url": "https://meet.google.com/abc-defg-hij"}))
|
||||
assert out["success"] is False
|
||||
assert "prerequisites missing" in out["error"]
|
||||
|
||||
|
||||
def test_meet_say_requires_text():
|
||||
from plugins.google_meet.tools import handle_meet_say
|
||||
|
||||
out = json.loads(handle_meet_say({}))
|
||||
assert out["success"] is False
|
||||
assert "text is required" in out["error"]
|
||||
|
||||
|
||||
def test_meet_say_no_active_meeting():
|
||||
from plugins.google_meet.tools import handle_meet_say
|
||||
|
||||
out = json.loads(handle_meet_say({"text": "hello everyone"}))
|
||||
assert out["success"] is False
|
||||
# Falls through to pm.enqueue_say which reports no active meeting.
|
||||
assert "no active meeting" in out.get("reason", "")
|
||||
|
||||
|
||||
def test_meet_status_and_transcript_no_active():
|
||||
from plugins.google_meet.tools import handle_meet_status, handle_meet_transcript
|
||||
|
||||
assert json.loads(handle_meet_status({}))["success"] is False
|
||||
assert json.loads(handle_meet_transcript({}))["success"] is False
|
||||
|
||||
|
||||
def test_meet_leave_no_active():
|
||||
from plugins.google_meet.tools import handle_meet_leave
|
||||
|
||||
out = json.loads(handle_meet_leave({}))
|
||||
assert out["success"] is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _on_session_end — defensive cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_on_session_end_noop_when_nothing_active():
|
||||
from plugins.google_meet import _on_session_end
|
||||
# Should not raise and should not call stop().
|
||||
with patch("plugins.google_meet.pm.stop") as stop_mock:
|
||||
_on_session_end()
|
||||
stop_mock.assert_not_called()
|
||||
|
||||
|
||||
def test_on_session_end_stops_live_bot():
|
||||
from plugins.google_meet import _on_session_end
|
||||
from plugins.google_meet import pm
|
||||
|
||||
with patch.object(pm, "status", return_value={"ok": True, "alive": True}), \
|
||||
patch.object(pm, "stop") as stop_mock:
|
||||
_on_session_end()
|
||||
stop_mock.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin register() — platform gating + tool registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_register_refuses_on_windows():
|
||||
import plugins.google_meet as plugin
|
||||
|
||||
calls = {"tools": [], "cli": [], "hooks": []}
|
||||
|
||||
class _Ctx:
|
||||
def register_tool(self, **kw): calls["tools"].append(kw["name"])
|
||||
def register_cli_command(self, **kw): calls["cli"].append(kw["name"])
|
||||
def register_hook(self, name, fn): calls["hooks"].append(name)
|
||||
|
||||
with patch.object(plugin.platform, "system", return_value="Windows"):
|
||||
plugin.register(_Ctx())
|
||||
|
||||
assert calls == {"tools": [], "cli": [], "hooks": []}
|
||||
|
||||
|
||||
def test_register_wires_tools_cli_and_hook_on_linux():
|
||||
import plugins.google_meet as plugin
|
||||
|
||||
calls = {"tools": [], "cli": [], "hooks": []}
|
||||
|
||||
class _Ctx:
|
||||
def register_tool(self, **kw): calls["tools"].append(kw["name"])
|
||||
def register_cli_command(self, **kw): calls["cli"].append(kw["name"])
|
||||
def register_hook(self, name, fn): calls["hooks"].append(name)
|
||||
|
||||
with patch.object(plugin.platform, "system", return_value="Linux"):
|
||||
plugin.register(_Ctx())
|
||||
|
||||
assert set(calls["tools"]) == {
|
||||
"meet_join", "meet_status", "meet_transcript", "meet_leave", "meet_say",
|
||||
}
|
||||
assert calls["cli"] == ["meet"]
|
||||
assert calls["hooks"] == ["on_session_end"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# v2: process_manager.enqueue_say + realtime-mode passthrough
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_enqueue_say_requires_text():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
assert pm.enqueue_say("")["ok"] is False
|
||||
assert pm.enqueue_say(" ")["ok"] is False
|
||||
|
||||
|
||||
def test_enqueue_say_no_active_meeting():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
res = pm.enqueue_say("hi team")
|
||||
assert res["ok"] is False
|
||||
assert "no active meeting" in res["reason"]
|
||||
|
||||
|
||||
def test_enqueue_say_rejects_transcribe_mode(tmp_path):
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
out_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
|
||||
out_dir.mkdir(parents=True)
|
||||
pm._write_active({
|
||||
"pid": 0, "meeting_id": "abc-defg-hij",
|
||||
"out_dir": str(out_dir), "url": "https://meet.google.com/abc-defg-hij",
|
||||
"started_at": 0, "mode": "transcribe",
|
||||
})
|
||||
res = pm.enqueue_say("hi team")
|
||||
assert res["ok"] is False
|
||||
assert "transcribe mode" in res["reason"]
|
||||
|
||||
|
||||
def test_enqueue_say_writes_jsonl_in_realtime_mode():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
out_dir = Path(os.environ["HERMES_HOME"]) / "workspace" / "meetings" / "abc-defg-hij"
|
||||
out_dir.mkdir(parents=True)
|
||||
pm._write_active({
|
||||
"pid": 0, "meeting_id": "abc-defg-hij",
|
||||
"out_dir": str(out_dir), "url": "https://meet.google.com/abc-defg-hij",
|
||||
"started_at": 0, "mode": "realtime",
|
||||
})
|
||||
res = pm.enqueue_say("hello everyone")
|
||||
assert res["ok"] is True
|
||||
assert "enqueued_id" in res
|
||||
|
||||
queue = out_dir / "say_queue.jsonl"
|
||||
assert queue.is_file()
|
||||
lines = [json.loads(ln) for ln in queue.read_text().splitlines() if ln.strip()]
|
||||
assert len(lines) == 1
|
||||
assert lines[0]["text"] == "hello everyone"
|
||||
|
||||
|
||||
def test_start_passes_mode_into_active_record():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
class _FakeProc:
|
||||
def __init__(self, pid): self.pid = pid
|
||||
|
||||
with patch.object(pm.subprocess, "Popen", return_value=_FakeProc(12345)), \
|
||||
patch.object(pm, "_pid_alive", return_value=False):
|
||||
res = pm.start(
|
||||
"https://meet.google.com/abc-defg-hij",
|
||||
mode="realtime",
|
||||
)
|
||||
assert res["ok"] is True
|
||||
assert res["mode"] == "realtime"
|
||||
assert pm._read_active()["mode"] == "realtime"
|
||||
|
||||
|
||||
def test_start_realtime_env_vars_threaded_through():
|
||||
from plugins.google_meet import process_manager as pm
|
||||
|
||||
class _FakeProc:
|
||||
def __init__(self, pid): self.pid = pid
|
||||
|
||||
captured_env = {}
|
||||
def _fake_popen(argv, **kwargs):
|
||||
captured_env.update(kwargs.get("env") or {})
|
||||
return _FakeProc(11111)
|
||||
|
||||
with patch.object(pm.subprocess, "Popen", side_effect=_fake_popen), \
|
||||
patch.object(pm, "_pid_alive", return_value=False):
|
||||
pm.start(
|
||||
"https://meet.google.com/abc-defg-hij",
|
||||
mode="realtime",
|
||||
realtime_model="gpt-realtime",
|
||||
realtime_voice="alloy",
|
||||
realtime_instructions="Be brief.",
|
||||
realtime_api_key="sk-test",
|
||||
)
|
||||
assert captured_env["HERMES_MEET_MODE"] == "realtime"
|
||||
assert captured_env["HERMES_MEET_REALTIME_MODEL"] == "gpt-realtime"
|
||||
assert captured_env["HERMES_MEET_REALTIME_VOICE"] == "alloy"
|
||||
assert captured_env["HERMES_MEET_REALTIME_INSTRUCTIONS"] == "Be brief."
|
||||
assert captured_env["HERMES_MEET_REALTIME_KEY"] == "sk-test"
|
||||
|
||||
|
||||
def test_meet_join_accepts_realtime_mode():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
|
||||
with patch("plugins.google_meet.tools.check_meet_requirements", return_value=True), \
|
||||
patch("plugins.google_meet.tools.pm.start", return_value={"ok": True, "meeting_id": "x-y-z"}) as start_mock:
|
||||
out = json.loads(handle_meet_join({
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"mode": "realtime",
|
||||
}))
|
||||
assert out["success"] is True
|
||||
assert start_mock.call_args.kwargs["mode"] == "realtime"
|
||||
|
||||
|
||||
def test_meet_join_rejects_bad_mode():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
|
||||
out = json.loads(handle_meet_join({
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"mode": "bogus",
|
||||
}))
|
||||
assert out["success"] is False
|
||||
assert "mode must be" in out["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# v3: NodeClient routing from tool handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_meet_join_unknown_node_returns_clear_error():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
|
||||
out = json.loads(handle_meet_join({
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"node": "my-mac",
|
||||
}))
|
||||
assert out["success"] is False
|
||||
assert "no registered meet node" in out["error"]
|
||||
|
||||
|
||||
def test_meet_join_routes_to_registered_node():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
reg = NodeRegistry()
|
||||
reg.add("my-mac", "ws://1.2.3.4:18789", "tok")
|
||||
|
||||
with patch("plugins.google_meet.node.client.NodeClient.start_bot",
|
||||
return_value={"ok": True, "meeting_id": "a-b-c"}) as call_mock:
|
||||
out = json.loads(handle_meet_join({
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"node": "my-mac",
|
||||
"mode": "realtime",
|
||||
}))
|
||||
assert out["success"] is True
|
||||
assert out["node"] == "my-mac"
|
||||
assert call_mock.call_args.kwargs["mode"] == "realtime"
|
||||
|
||||
|
||||
def test_meet_say_routes_to_node():
|
||||
from plugins.google_meet.tools import handle_meet_say
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
reg = NodeRegistry()
|
||||
reg.add("my-mac", "ws://1.2.3.4:18789", "tok")
|
||||
|
||||
with patch("plugins.google_meet.node.client.NodeClient.say",
|
||||
return_value={"ok": True, "enqueued_id": "abc"}) as call_mock:
|
||||
out = json.loads(handle_meet_say({"text": "hello", "node": "my-mac"}))
|
||||
assert out["success"] is True
|
||||
assert out["node"] == "my-mac"
|
||||
call_mock.assert_called_once_with("hello")
|
||||
|
||||
|
||||
def test_meet_join_auto_node_selects_sole_registered():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
reg = NodeRegistry()
|
||||
reg.add("only-one", "ws://1.2.3.4:18789", "tok")
|
||||
|
||||
with patch("plugins.google_meet.node.client.NodeClient.start_bot",
|
||||
return_value={"ok": True}) as call_mock:
|
||||
out = json.loads(handle_meet_join({
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"node": "auto",
|
||||
}))
|
||||
assert out["success"] is True
|
||||
assert out["node"] == "only-one"
|
||||
assert call_mock.called
|
||||
|
||||
|
||||
def test_meet_join_auto_node_ambiguous_returns_error():
|
||||
from plugins.google_meet.tools import handle_meet_join
|
||||
from plugins.google_meet.node.registry import NodeRegistry
|
||||
|
||||
reg = NodeRegistry()
|
||||
reg.add("a", "ws://1.2.3.4:18789", "tok")
|
||||
reg.add("b", "ws://5.6.7.8:18789", "tok")
|
||||
|
||||
out = json.loads(handle_meet_join({
|
||||
"url": "https://meet.google.com/abc-defg-hij",
|
||||
"node": "auto",
|
||||
}))
|
||||
assert out["success"] is False
|
||||
assert "no registered meet node" in out["error"]
|
||||
|
||||
|
||||
def test_cli_register_includes_node_subcommand():
|
||||
"""`hermes meet` argparse tree includes the node subtree."""
|
||||
import argparse
|
||||
from plugins.google_meet.cli import register_cli
|
||||
|
||||
parser = argparse.ArgumentParser(prog="hermes meet")
|
||||
register_cli(parser)
|
||||
|
||||
# Parse a known-good node invocation to prove the subtree is wired.
|
||||
ns = parser.parse_args(["node", "list"])
|
||||
assert ns.meet_command == "node"
|
||||
assert ns.node_cmd == "list"
|
||||
|
||||
|
||||
def test_cli_join_accepts_mode_and_node_flags():
|
||||
import argparse
|
||||
from plugins.google_meet.cli import register_cli
|
||||
|
||||
parser = argparse.ArgumentParser(prog="hermes meet")
|
||||
register_cli(parser)
|
||||
|
||||
ns = parser.parse_args([
|
||||
"join", "https://meet.google.com/abc-defg-hij",
|
||||
"--mode", "realtime", "--node", "my-mac",
|
||||
])
|
||||
assert ns.mode == "realtime"
|
||||
assert ns.node == "my-mac"
|
||||
|
||||
|
||||
def test_cli_say_subcommand_exists():
|
||||
import argparse
|
||||
from plugins.google_meet.cli import register_cli
|
||||
|
||||
parser = argparse.ArgumentParser(prog="hermes meet")
|
||||
register_cli(parser)
|
||||
|
||||
ns = parser.parse_args(["say", "hello team", "--node", "my-mac"])
|
||||
assert ns.text == "hello team"
|
||||
assert ns.node == "my-mac"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# v2.1: new _BotState fields + status dict shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_bot_state_exposes_v2_telemetry_fields(tmp_path):
|
||||
from plugins.google_meet.meet_bot import _BotState
|
||||
|
||||
state = _BotState(out_dir=tmp_path / "s", meeting_id="x-y-z",
|
||||
url="https://meet.google.com/x-y-z")
|
||||
# Defaults for the new fields.
|
||||
status = json.loads((tmp_path / "s" / "status.json").read_text())
|
||||
for key in (
|
||||
"realtime", "realtimeReady", "realtimeDevice",
|
||||
"audioBytesOut", "lastAudioOutAt", "lastBargeInAt",
|
||||
"joinAttemptedAt", "leaveReason",
|
||||
):
|
||||
assert key in status, f"missing v2 telemetry key: {key}"
|
||||
assert status["realtime"] is False
|
||||
assert status["realtimeReady"] is False
|
||||
assert status["audioBytesOut"] == 0
|
||||
|
||||
# Setting them flushes them.
|
||||
state.set(realtime=True, realtime_ready=True, audio_bytes_out=1024,
|
||||
leave_reason="lobby_timeout")
|
||||
status = json.loads((tmp_path / "s" / "status.json").read_text())
|
||||
assert status["realtime"] is True
|
||||
assert status["realtimeReady"] is True
|
||||
assert status["audioBytesOut"] == 1024
|
||||
assert status["leaveReason"] == "lobby_timeout"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admission detection + barge-in helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_looks_like_human_speaker():
|
||||
from plugins.google_meet.meet_bot import _looks_like_human_speaker
|
||||
|
||||
# Blank, "unknown", "you", and the bot's own name → not human (no barge-in)
|
||||
for s in ("", " ", "Unknown", "unknown", "You", "you", "Hermes Agent", "hermes agent"):
|
||||
assert not _looks_like_human_speaker(s, "Hermes Agent"), f"{s!r} should NOT be human"
|
||||
# Real names → human (barge-in)
|
||||
for s in ("Alice", "Bob Lee", "@teknium"):
|
||||
assert _looks_like_human_speaker(s, "Hermes Agent"), f"{s!r} SHOULD be human"
|
||||
|
||||
|
||||
def test_detect_admission_returns_false_on_error():
|
||||
from plugins.google_meet.meet_bot import _detect_admission
|
||||
|
||||
class _FakePage:
|
||||
def evaluate(self, _js): raise RuntimeError("boom")
|
||||
|
||||
assert _detect_admission(_FakePage()) is False
|
||||
|
||||
|
||||
def test_detect_admission_true_when_probe_returns_true():
|
||||
from plugins.google_meet.meet_bot import _detect_admission
|
||||
|
||||
class _FakePage:
|
||||
def evaluate(self, _js): return True
|
||||
|
||||
assert _detect_admission(_FakePage()) is True
|
||||
|
||||
|
||||
def test_detect_denied_returns_false_on_error():
|
||||
from plugins.google_meet.meet_bot import _detect_denied
|
||||
|
||||
class _FakePage:
|
||||
def evaluate(self, _js): raise RuntimeError("boom")
|
||||
|
||||
assert _detect_denied(_FakePage()) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Realtime session counters + cancel_response (barge-in)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_realtime_session_cancel_response_when_disconnected():
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
|
||||
# No _ws yet — cancel should no-op and return False.
|
||||
assert sess.cancel_response() is False
|
||||
|
||||
|
||||
def test_realtime_session_cancel_response_sends_cancel_frame():
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
|
||||
sent = []
|
||||
|
||||
class _FakeWs:
|
||||
def send(self, msg): sent.append(msg)
|
||||
|
||||
sess._ws = _FakeWs()
|
||||
assert sess.cancel_response() is True
|
||||
assert len(sent) == 1
|
||||
import json as _j
|
||||
envelope = _j.loads(sent[0])
|
||||
assert envelope == {"type": "response.cancel"}
|
||||
|
||||
|
||||
def test_realtime_session_counters_initialized():
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test", audio_sink_path=None)
|
||||
assert sess.audio_bytes_out == 0
|
||||
assert sess.last_audio_out_at is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hermes meet install CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_cli_install_subcommand_is_registered():
|
||||
import argparse
|
||||
from plugins.google_meet.cli import register_cli
|
||||
|
||||
parser = argparse.ArgumentParser(prog="hermes meet")
|
||||
register_cli(parser)
|
||||
|
||||
ns = parser.parse_args(["install"])
|
||||
assert ns.meet_command == "install"
|
||||
assert ns.realtime is False
|
||||
assert ns.yes is False
|
||||
|
||||
|
||||
def test_cli_install_flags_parse():
|
||||
import argparse
|
||||
from plugins.google_meet.cli import register_cli
|
||||
|
||||
parser = argparse.ArgumentParser(prog="hermes meet")
|
||||
register_cli(parser)
|
||||
|
||||
ns = parser.parse_args(["install", "--realtime", "--yes"])
|
||||
assert ns.realtime is True
|
||||
assert ns.yes is True
|
||||
|
||||
|
||||
def test_cmd_install_refuses_windows(capsys):
|
||||
from plugins.google_meet.cli import _cmd_install
|
||||
|
||||
with patch("plugins.google_meet.cli.platform" if False else "platform.system",
|
||||
return_value="Windows"):
|
||||
rc = _cmd_install(realtime=False, assume_yes=True)
|
||||
assert rc == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "Windows" in out
|
||||
|
||||
|
||||
def test_cmd_install_runs_pip_and_playwright(capsys):
|
||||
"""End-to-end wiring: pip + playwright install invoked, returncodes handled."""
|
||||
from plugins.google_meet.cli import _cmd_install
|
||||
|
||||
calls = []
|
||||
class _FakeRes:
|
||||
def __init__(self, rc=0): self.returncode = rc
|
||||
|
||||
def _fake_run(argv, **kwargs):
|
||||
calls.append(list(argv))
|
||||
return _FakeRes(0)
|
||||
|
||||
with patch("platform.system", return_value="Linux"), \
|
||||
patch("subprocess.run", side_effect=_fake_run), \
|
||||
patch("shutil.which", return_value="/usr/bin/paplay"):
|
||||
rc = _cmd_install(realtime=False, assume_yes=True)
|
||||
assert rc == 0
|
||||
# First invocation: pip install
|
||||
pip_cmds = [c for c in calls if len(c) > 2 and c[1:4] == ["-m", "pip", "install"]]
|
||||
assert pip_cmds, f"no pip install run: {calls}"
|
||||
assert "playwright" in pip_cmds[0]
|
||||
assert "websockets" in pip_cmds[0]
|
||||
# Second: playwright install chromium
|
||||
pw_cmds = [c for c in calls if len(c) > 2 and c[1:4] == ["-m", "playwright", "install"]]
|
||||
assert pw_cmds, f"no playwright install run: {calls}"
|
||||
assert "chromium" in pw_cmds[0]
|
||||
|
||||
|
||||
def test_cmd_install_realtime_skips_when_deps_present(capsys):
|
||||
"""When paplay + pactl are already on PATH, no sudo call happens."""
|
||||
from plugins.google_meet.cli import _cmd_install
|
||||
|
||||
calls = []
|
||||
class _FakeRes:
|
||||
def __init__(self, rc=0): self.returncode = rc
|
||||
|
||||
def _fake_run(argv, **kwargs):
|
||||
calls.append(list(argv))
|
||||
return _FakeRes(0)
|
||||
|
||||
with patch("platform.system", return_value="Linux"), \
|
||||
patch("subprocess.run", side_effect=_fake_run), \
|
||||
patch("shutil.which", return_value="/usr/bin/paplay"):
|
||||
rc = _cmd_install(realtime=True, assume_yes=True)
|
||||
assert rc == 0
|
||||
# No sudo apt-get call — paplay was already on PATH.
|
||||
sudo_calls = [c for c in calls if c and c[0] == "sudo"]
|
||||
assert sudo_calls == [], f"unexpected sudo invocation: {sudo_calls}"
|
||||
out = capsys.readouterr().out
|
||||
assert "already installed" in out
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Tests for plugins.google_meet.realtime.openai_client (v2).
|
||||
|
||||
Uses a scripted fake WebSocket — no network, no API key required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_home(tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
yield hermes_home
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake WebSocket
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeWS:
|
||||
"""Scripted WS: send() records frames, recv() pops a queue."""
|
||||
|
||||
def __init__(self, recv_frames: list):
|
||||
self.sent: list[dict] = []
|
||||
self._recv_q: list = list(recv_frames)
|
||||
self.closed = False
|
||||
|
||||
def send(self, payload):
|
||||
# Always accept str payloads — client encodes JSON with json.dumps.
|
||||
if isinstance(payload, (bytes, bytearray)):
|
||||
payload = payload.decode()
|
||||
self.sent.append(json.loads(payload))
|
||||
|
||||
def recv(self, timeout=None): # noqa: ARG002
|
||||
if not self._recv_q:
|
||||
raise RuntimeError("fake ws: no more frames")
|
||||
frame = self._recv_q.pop(0)
|
||||
if isinstance(frame, dict):
|
||||
return json.dumps(frame)
|
||||
return frame
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _install_fake_websockets(monkeypatch, fake_ws):
|
||||
"""Install a fake ``websockets.sync.client`` module in sys.modules."""
|
||||
mod_websockets = types.ModuleType("websockets")
|
||||
mod_sync = types.ModuleType("websockets.sync")
|
||||
mod_sync_client = types.ModuleType("websockets.sync.client")
|
||||
|
||||
captured = {"url": None, "headers": None, "kwargs": None}
|
||||
|
||||
def _connect(url, **kwargs):
|
||||
captured["url"] = url
|
||||
captured["kwargs"] = kwargs
|
||||
captured["headers"] = (
|
||||
kwargs.get("additional_headers") or kwargs.get("extra_headers")
|
||||
)
|
||||
return fake_ws
|
||||
|
||||
mod_sync_client.connect = _connect
|
||||
mod_sync.client = mod_sync_client
|
||||
mod_websockets.sync = mod_sync
|
||||
|
||||
monkeypatch.setitem(sys.modules, "websockets", mod_websockets)
|
||||
monkeypatch.setitem(sys.modules, "websockets.sync", mod_sync)
|
||||
monkeypatch.setitem(sys.modules, "websockets.sync.client", mod_sync_client)
|
||||
return captured
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# connect()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_connect_sends_session_update_with_voice_and_instructions(monkeypatch):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
ws = _FakeWS(recv_frames=[])
|
||||
captured = _install_fake_websockets(monkeypatch, ws)
|
||||
|
||||
sess = RealtimeSession(
|
||||
api_key="sk-test",
|
||||
model="gpt-realtime",
|
||||
voice="verse",
|
||||
instructions="Be brief.",
|
||||
)
|
||||
sess.connect()
|
||||
|
||||
# Auth + beta headers set.
|
||||
assert captured["url"].startswith("wss://api.openai.com/v1/realtime")
|
||||
assert "model=gpt-realtime" in captured["url"]
|
||||
headers = captured["headers"] or []
|
||||
hdict = dict(headers)
|
||||
assert hdict.get("Authorization") == "Bearer sk-test"
|
||||
assert hdict.get("OpenAI-Beta") == "realtime=v1"
|
||||
|
||||
# First frame sent must be session.update with the right shape.
|
||||
assert len(ws.sent) == 1
|
||||
update = ws.sent[0]
|
||||
assert update["type"] == "session.update"
|
||||
s = update["session"]
|
||||
assert s["voice"] == "verse"
|
||||
assert s["instructions"] == "Be brief."
|
||||
assert set(s["modalities"]) == {"audio", "text"}
|
||||
assert s["output_audio_format"] == "pcm16"
|
||||
assert s["input_audio_format"] == "pcm16"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# speak()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_speak_sends_create_and_response_and_writes_audio(monkeypatch, tmp_path):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
audio_bytes = b"\x01\x02\x03\x04PCM!"
|
||||
b64 = base64.b64encode(audio_bytes).decode()
|
||||
|
||||
recv_frames = [
|
||||
{"type": "response.created"},
|
||||
{"type": "response.audio.delta", "delta": b64},
|
||||
{"type": "response.audio.delta", "delta": base64.b64encode(b"more").decode()},
|
||||
{"type": "response.done"},
|
||||
]
|
||||
ws = _FakeWS(recv_frames=recv_frames)
|
||||
_install_fake_websockets(monkeypatch, ws)
|
||||
|
||||
sink = tmp_path / "out.pcm"
|
||||
sess = RealtimeSession(api_key="sk-test", audio_sink_path=sink)
|
||||
sess.connect()
|
||||
result = sess.speak("Hello everyone.")
|
||||
|
||||
# Frames sent after session.update: conversation.item.create then response.create.
|
||||
types_sent = [f["type"] for f in ws.sent]
|
||||
assert types_sent == ["session.update", "conversation.item.create", "response.create"]
|
||||
|
||||
item = ws.sent[1]["item"]
|
||||
assert item["role"] == "user"
|
||||
assert item["content"][0]["type"] == "input_text"
|
||||
assert item["content"][0]["text"] == "Hello everyone."
|
||||
|
||||
resp = ws.sent[2]["response"]
|
||||
assert resp["modalities"] == ["audio"]
|
||||
|
||||
# Audio file got decoded + appended bytes.
|
||||
data = sink.read_bytes()
|
||||
assert data == audio_bytes + b"more"
|
||||
assert result["ok"] is True
|
||||
assert result["bytes_written"] == len(audio_bytes) + len(b"more")
|
||||
assert result["duration_ms"] >= 0.0
|
||||
|
||||
|
||||
def test_speak_raises_on_error_frame(monkeypatch, tmp_path):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
ws = _FakeWS(recv_frames=[
|
||||
{"type": "response.created"},
|
||||
{"type": "error", "error": {"message": "bad juju"}},
|
||||
])
|
||||
_install_fake_websockets(monkeypatch, ws)
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test", audio_sink_path=tmp_path / "o.pcm")
|
||||
sess.connect()
|
||||
with pytest.raises(RuntimeError, match="bad juju"):
|
||||
sess.speak("hi")
|
||||
|
||||
|
||||
def test_speak_without_connect_raises(monkeypatch):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test")
|
||||
with pytest.raises(RuntimeError, match="connect"):
|
||||
sess.speak("hi")
|
||||
|
||||
|
||||
def test_close_is_idempotent_and_closes_ws(monkeypatch):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
ws = _FakeWS(recv_frames=[])
|
||||
_install_fake_websockets(monkeypatch, ws)
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test")
|
||||
sess.connect()
|
||||
sess.close()
|
||||
assert ws.closed is True
|
||||
# Second close is a no-op.
|
||||
sess.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# websockets dependency missing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_connect_raises_clean_error_when_websockets_missing(monkeypatch):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSession
|
||||
|
||||
# Make `import websockets.sync.client` fail.
|
||||
monkeypatch.setitem(sys.modules, "websockets", None)
|
||||
monkeypatch.setitem(sys.modules, "websockets.sync", None)
|
||||
monkeypatch.setitem(sys.modules, "websockets.sync.client", None)
|
||||
|
||||
sess = RealtimeSession(api_key="sk-test")
|
||||
with pytest.raises(RuntimeError, match="pip install websockets"):
|
||||
sess.connect()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RealtimeSpeaker
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _StubSession:
|
||||
def __init__(self):
|
||||
self.spoken: list[str] = []
|
||||
|
||||
def speak(self, text, timeout=30.0): # noqa: ARG002
|
||||
self.spoken.append(text)
|
||||
return {"ok": True, "bytes_written": len(text), "duration_ms": 1.0}
|
||||
|
||||
|
||||
def test_speaker_run_until_stopped_processes_queue(tmp_path):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
|
||||
|
||||
queue = tmp_path / "queue.jsonl"
|
||||
processed = tmp_path / "processed.jsonl"
|
||||
queue.write_text(
|
||||
json.dumps({"id": "a", "text": "hello one"}) + "\n"
|
||||
+ json.dumps({"id": "b", "text": "hello two"}) + "\n"
|
||||
)
|
||||
|
||||
stub = _StubSession()
|
||||
speaker = RealtimeSpeaker(stub, queue_path=queue, processed_path=processed)
|
||||
|
||||
# Stop once the queue is empty.
|
||||
def _stop():
|
||||
return queue.exists() and queue.read_text().strip() == ""
|
||||
|
||||
speaker.run_until_stopped(_stop, poll_interval=0.01)
|
||||
|
||||
assert stub.spoken == ["hello one", "hello two"]
|
||||
|
||||
# Processed file has both entries, in order.
|
||||
lines = [json.loads(l) for l in processed.read_text().splitlines() if l.strip()]
|
||||
assert [l["id"] for l in lines] == ["a", "b"]
|
||||
assert all(l["result"]["ok"] for l in lines)
|
||||
|
||||
# Queue is empty (possibly empty string) after processing.
|
||||
assert queue.read_text().strip() == ""
|
||||
|
||||
|
||||
def test_speaker_exits_immediately_when_stop_fn_true(tmp_path):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
|
||||
|
||||
queue = tmp_path / "q.jsonl"
|
||||
queue.write_text(json.dumps({"id": "x", "text": "never spoken"}) + "\n")
|
||||
|
||||
stub = _StubSession()
|
||||
speaker = RealtimeSpeaker(stub, queue_path=queue)
|
||||
speaker.run_until_stopped(lambda: True, poll_interval=0.01)
|
||||
assert stub.spoken == []
|
||||
|
||||
|
||||
def test_speaker_drops_line_without_processed_path_when_none(tmp_path):
|
||||
from plugins.google_meet.realtime.openai_client import RealtimeSpeaker
|
||||
|
||||
queue = tmp_path / "q.jsonl"
|
||||
queue.write_text(json.dumps({"id": "only", "text": "once"}) + "\n")
|
||||
|
||||
stub = _StubSession()
|
||||
speaker = RealtimeSpeaker(stub, queue_path=queue, processed_path=None)
|
||||
|
||||
def _stop():
|
||||
return queue.read_text().strip() == ""
|
||||
|
||||
speaker.run_until_stopped(_stop, poll_interval=0.01)
|
||||
assert stub.spoken == ["once"]
|
||||
assert queue.read_text().strip() == ""
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Tests for Kanban task file attachments (#35338).
|
||||
|
||||
Covers three layers:
|
||||
* ``hermes_cli.kanban_db`` accessors (add/list/get/delete + path helpers)
|
||||
* the dashboard REST surface (upload / list / download / delete)
|
||||
* worker-context surfacing so a kanban worker sees the absolute paths
|
||||
|
||||
The plugin router is attached to a bare FastAPI app — same approach as
|
||||
``test_kanban_dashboard_plugin.py`` — so we exercise the real HTTP path
|
||||
(multipart upload, streaming download) without the whole dashboard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from hermes_cli import kanban_db as kb
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_plugin_router():
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
plugin_file = repo_root / "plugins" / "kanban" / "dashboard" / "plugin_api.py"
|
||||
assert plugin_file.exists(), f"plugin file missing: {plugin_file}"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"hermes_dashboard_plugin_kanban_attach_test", plugin_file,
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod.router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kanban_home(tmp_path, monkeypatch):
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
kb.init_db()
|
||||
return home
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(kanban_home):
|
||||
app = FastAPI()
|
||||
app.include_router(_load_plugin_router(), prefix="/api/plugins/kanban")
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _make_task(conn, title="t") -> str:
|
||||
return kb.create_task(conn, title=title)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB-layer accessors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_add_list_get_delete_attachment(kanban_home, tmp_path):
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = _make_task(conn)
|
||||
# Write a real blob under the per-task dir so delete can unlink it.
|
||||
dest_dir = kb.task_attachments_dir(task_id)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
blob = dest_dir / "source.pdf"
|
||||
blob.write_bytes(b"%PDF-1.4 fake")
|
||||
|
||||
att_id = kb.add_attachment(
|
||||
conn,
|
||||
task_id,
|
||||
filename="source.pdf",
|
||||
stored_path=str(blob),
|
||||
content_type="application/pdf",
|
||||
size=blob.stat().st_size,
|
||||
uploaded_by="tester",
|
||||
)
|
||||
assert att_id > 0
|
||||
|
||||
atts = kb.list_attachments(conn, task_id)
|
||||
assert len(atts) == 1
|
||||
a = atts[0]
|
||||
assert a.filename == "source.pdf"
|
||||
assert a.content_type == "application/pdf"
|
||||
assert a.size == len(b"%PDF-1.4 fake")
|
||||
assert a.uploaded_by == "tester"
|
||||
assert a.stored_path == str(blob)
|
||||
|
||||
got = kb.get_attachment(conn, att_id)
|
||||
assert got is not None and got.id == att_id
|
||||
|
||||
removed = kb.delete_attachment(conn, att_id)
|
||||
assert removed is not None and removed.id == att_id
|
||||
assert kb.list_attachments(conn, task_id) == []
|
||||
assert not blob.exists(), "delete should unlink the on-disk blob"
|
||||
assert kb.get_attachment(conn, att_id) is None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_add_attachment_rejects_unknown_task(kanban_home):
|
||||
conn = kb.connect()
|
||||
try:
|
||||
with pytest.raises(ValueError):
|
||||
kb.add_attachment(
|
||||
conn, "t_doesnotexist", filename="x.txt", stored_path="/tmp/x.txt"
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_add_attachment_appends_event(kanban_home):
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = _make_task(conn)
|
||||
kb.add_attachment(
|
||||
conn, task_id, filename="a.txt", stored_path="/tmp/a.txt", size=3
|
||||
)
|
||||
kinds = [e.kind for e in kb.list_events(conn, task_id)]
|
||||
assert "attached" in kinds
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_delete_attachment_missing_returns_none(kanban_home):
|
||||
conn = kb.connect()
|
||||
try:
|
||||
assert kb.delete_attachment(conn, 999999) is None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_attachments_root_is_per_board(kanban_home, monkeypatch):
|
||||
# default board uses <root>/kanban/attachments
|
||||
default_root = kb.attachments_root(board="default")
|
||||
assert default_root.name == "attachments"
|
||||
# a named board nests under its board dir
|
||||
monkeypatch.delenv("HERMES_KANBAN_ATTACHMENTS_ROOT", raising=False)
|
||||
named = kb.attachments_root(board="default")
|
||||
assert named == default_root
|
||||
|
||||
|
||||
def test_attachments_root_env_override(kanban_home, monkeypatch, tmp_path):
|
||||
override = tmp_path / "custom-attach"
|
||||
monkeypatch.setenv("HERMES_KANBAN_ATTACHMENTS_ROOT", str(override))
|
||||
assert kb.attachments_root() == override
|
||||
assert kb.task_attachments_dir("t_abc") == override / "t_abc"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker context surfacing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_worker_context_lists_attachments_with_absolute_path(kanban_home):
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = _make_task(conn, title="translate PDF")
|
||||
dest_dir = kb.task_attachments_dir(task_id)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
blob = dest_dir / "manual.pdf"
|
||||
blob.write_bytes(b"data")
|
||||
kb.add_attachment(
|
||||
conn,
|
||||
task_id,
|
||||
filename="manual.pdf",
|
||||
stored_path=str(blob.resolve()),
|
||||
content_type="application/pdf",
|
||||
size=4,
|
||||
)
|
||||
ctx = kb.build_worker_context(conn, task_id)
|
||||
assert "## Attachments" in ctx
|
||||
assert "manual.pdf" in ctx
|
||||
# The absolute path must appear so the worker can read_file it.
|
||||
assert str(blob.resolve()) in ctx
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_worker_context_no_attachments_section_when_empty(kanban_home):
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = _make_task(conn)
|
||||
ctx = kb.build_worker_context(conn, task_id)
|
||||
assert "## Attachments" not in ctx
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# REST surface — upload / list / download / delete round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_task_via_api(client) -> str:
|
||||
r = client.post("/api/plugins/kanban/tasks", json={"title": "x"})
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()["task"]["id"]
|
||||
|
||||
|
||||
def test_upload_list_download_delete_roundtrip(client):
|
||||
task_id = _create_task_via_api(client)
|
||||
content = b"hello attachment world"
|
||||
|
||||
# Upload
|
||||
r = client.post(
|
||||
f"/api/plugins/kanban/tasks/{task_id}/attachments",
|
||||
files={"file": ("notes.txt", content, "text/plain")},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
att = r.json()["attachment"]
|
||||
assert att["filename"] == "notes.txt"
|
||||
assert att["size"] == len(content)
|
||||
att_id = att["id"]
|
||||
|
||||
# List (drawer also embeds it in GET /tasks/:id)
|
||||
r = client.get(f"/api/plugins/kanban/tasks/{task_id}/attachments")
|
||||
assert r.status_code == 200
|
||||
assert [a["filename"] for a in r.json()["attachments"]] == ["notes.txt"]
|
||||
|
||||
detail = client.get(f"/api/plugins/kanban/tasks/{task_id}").json()
|
||||
assert "attachments" in detail
|
||||
assert len(detail["attachments"]) == 1
|
||||
|
||||
# Download streams the exact bytes back
|
||||
r = client.get(f"/api/plugins/kanban/attachments/{att_id}")
|
||||
assert r.status_code == 200
|
||||
assert r.content == content
|
||||
|
||||
# Delete removes the row and the file
|
||||
r = client.delete(f"/api/plugins/kanban/attachments/{att_id}")
|
||||
assert r.status_code == 200
|
||||
assert client.get(f"/api/plugins/kanban/attachments/{att_id}").status_code == 404
|
||||
assert client.get(
|
||||
f"/api/plugins/kanban/tasks/{task_id}/attachments"
|
||||
).json()["attachments"] == []
|
||||
|
||||
|
||||
def test_upload_sanitizes_traversal_filename(client):
|
||||
task_id = _create_task_via_api(client)
|
||||
r = client.post(
|
||||
f"/api/plugins/kanban/tasks/{task_id}/attachments",
|
||||
files={"file": ("../../../../etc/passwd", b"x", "text/plain")},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
stored_path = r.json()["attachment"]["stored_path"]
|
||||
# The leaf name only; never escapes the per-task attachments dir.
|
||||
assert Path(stored_path).name == "passwd"
|
||||
task_dir = kb.task_attachments_dir(task_id).resolve()
|
||||
assert Path(stored_path).resolve().is_relative_to(task_dir)
|
||||
|
||||
|
||||
def test_upload_name_collision_gets_suffixed(client):
|
||||
task_id = _create_task_via_api(client)
|
||||
for _ in range(2):
|
||||
r = client.post(
|
||||
f"/api/plugins/kanban/tasks/{task_id}/attachments",
|
||||
files={"file": ("dup.txt", b"a", "text/plain")},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
names = sorted(
|
||||
a["filename"]
|
||||
for a in client.get(
|
||||
f"/api/plugins/kanban/tasks/{task_id}/attachments"
|
||||
).json()["attachments"]
|
||||
)
|
||||
assert names == ["dup (1).txt", "dup.txt"]
|
||||
|
||||
|
||||
def test_upload_unknown_task_404(client):
|
||||
r = client.post(
|
||||
"/api/plugins/kanban/tasks/t_nope/attachments",
|
||||
files={"file": ("x.txt", b"x", "text/plain")},
|
||||
)
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_download_unknown_attachment_404(client):
|
||||
assert client.get("/api/plugins/kanban/attachments/424242").status_code == 404
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,440 @@
|
||||
"""Tests for kanban worker/runs read endpoints.
|
||||
|
||||
Covers:
|
||||
GET /workers/active
|
||||
GET /runs/{run_id}
|
||||
GET /runs/{run_id}/inspect
|
||||
POST /runs/{run_id}/terminate
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import secrets
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from hermes_cli import kanban_db as kb
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_plugin_router():
|
||||
"""Dynamically load plugins/kanban/dashboard/plugin_api.py and return its router."""
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
plugin_file = repo_root / "plugins" / "kanban" / "dashboard" / "plugin_api.py"
|
||||
assert plugin_file.exists(), f"plugin file missing: {plugin_file}"
|
||||
|
||||
mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
|
||||
# Re-use a cached module if already loaded to avoid duplicate-router issues.
|
||||
if mod_name in sys.modules:
|
||||
return sys.modules[mod_name].router
|
||||
|
||||
spec = importlib.util.spec_from_file_location(mod_name, plugin_file)
|
||||
assert spec is not None and spec.loader is not None
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[mod_name] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod.router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kanban_home(tmp_path, monkeypatch):
|
||||
"""Isolated HERMES_HOME with an empty kanban DB."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
kb.init_db()
|
||||
return home
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(kanban_home):
|
||||
app = FastAPI()
|
||||
app.include_router(_load_plugin_router(), prefix="/api/plugins/kanban")
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _insert_run(conn, task_id, *, worker_pid=None, ended_at=None):
|
||||
"""Insert a task_runs row directly (bypassing claim machinery) and return run_id."""
|
||||
lock = secrets.token_hex(8)
|
||||
future = int(time.time()) + 3600
|
||||
cur = conn.execute(
|
||||
"INSERT INTO task_runs "
|
||||
"(task_id, status, claim_lock, claim_expires, worker_pid, started_at, ended_at) "
|
||||
"VALUES (?, 'running', ?, ?, ?, ?, ?)",
|
||||
(task_id, lock, future, worker_pid, int(time.time()), ended_at),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /workers/active
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_workers_active_empty_board(client):
|
||||
"""Board with no running tasks returns an empty workers list."""
|
||||
r = client.get("/api/plugins/kanban/workers/active")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["workers"] == []
|
||||
assert body["count"] == 0
|
||||
assert "checked_at" in body
|
||||
|
||||
|
||||
def test_workers_active_with_running_task(client):
|
||||
"""A running task with an open run row and worker_pid appears in the list."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="active-worker", assignee="alice")
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status='running' WHERE id=?", (task_id,),
|
||||
)
|
||||
_insert_run(conn, task_id, worker_pid=12345)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.get("/api/plugins/kanban/workers/active")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["count"] == 1
|
||||
w = body["workers"][0]
|
||||
assert w["task_id"] == task_id
|
||||
assert w["worker_pid"] == 12345
|
||||
assert w["task_status"] == "running"
|
||||
assert w["task_title"] == "active-worker"
|
||||
assert w["task_assignee"] == "alice"
|
||||
|
||||
|
||||
def test_workers_active_excludes_ended_runs(client):
|
||||
"""Runs with ended_at set are excluded even if task is running."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="ended-run", assignee="bob")
|
||||
conn.execute("UPDATE tasks SET status='running' WHERE id=?", (task_id,))
|
||||
_insert_run(conn, task_id, worker_pid=99999, ended_at=int(time.time()) - 60)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.get("/api/plugins/kanban/workers/active")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["count"] == 0
|
||||
|
||||
|
||||
def test_workers_active_excludes_runs_without_pid(client):
|
||||
"""Runs with no worker_pid are not considered active workers."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="no-pid", assignee="carol")
|
||||
conn.execute("UPDATE tasks SET status='running' WHERE id=?", (task_id,))
|
||||
_insert_run(conn, task_id, worker_pid=None)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.get("/api/plugins/kanban/workers/active")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["count"] == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /runs/{run_id}
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_get_run_404_unknown_id(client):
|
||||
"""Non-existent run_id returns 404."""
|
||||
r = client.get("/api/plugins/kanban/runs/999999")
|
||||
assert r.status_code == 404
|
||||
assert "999999" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_get_run_ok(client):
|
||||
"""Existing run row returns 200 with expected shape."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="run-lookup", assignee="dave")
|
||||
run_id = _insert_run(conn, task_id, worker_pid=55555)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.get(f"/api/plugins/kanban/runs/{run_id}")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "run" in body
|
||||
run = body["run"]
|
||||
assert run["id"] == run_id
|
||||
assert run["task_id"] == task_id
|
||||
assert run["worker_pid"] == 55555
|
||||
assert run["ended_at"] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /runs/{run_id}/inspect
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_inspect_run_404(client):
|
||||
"""Non-existent run_id returns 404."""
|
||||
r = client.get("/api/plugins/kanban/runs/888888/inspect")
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_inspect_run_already_ended(client):
|
||||
"""Run with ended_at set returns alive=false with reason."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="ended", assignee="eve")
|
||||
run_id = _insert_run(conn, task_id, worker_pid=11111, ended_at=int(time.time()) - 10)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["alive"] is False
|
||||
assert "ended" in body["reason"]
|
||||
|
||||
|
||||
def test_inspect_run_no_pid(client):
|
||||
"""Run with no worker_pid returns alive=false with reason."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="no-pid-inspect", assignee="frank")
|
||||
run_id = _insert_run(conn, task_id, worker_pid=None)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["alive"] is False
|
||||
assert "worker_pid" in body["reason"]
|
||||
|
||||
|
||||
def test_inspect_run_dead_pid(client, monkeypatch):
|
||||
"""Run with a non-existent PID returns alive=false via psutil.NoSuchProcess."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="dead-pid", assignee="grace")
|
||||
run_id = _insert_run(conn, task_id, worker_pid=999999)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Mock psutil to raise NoSuchProcess for any PID.
|
||||
mock_psutil = MagicMock()
|
||||
mock_psutil.NoSuchProcess = Exception
|
||||
mock_psutil.AccessDenied = PermissionError
|
||||
|
||||
def _raise_no_such(*args, **kwargs):
|
||||
raise mock_psutil.NoSuchProcess("no such process")
|
||||
|
||||
mock_psutil.Process = _raise_no_such
|
||||
|
||||
# Patch the module-level _psutil in the loaded plugin module.
|
||||
plugin_mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
|
||||
plugin_mod = sys.modules.get(plugin_mod_name)
|
||||
if plugin_mod is not None:
|
||||
monkeypatch.setattr(plugin_mod, "_psutil", mock_psutil)
|
||||
else:
|
||||
pytest.skip("plugin module not yet loaded")
|
||||
|
||||
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["alive"] is False
|
||||
assert body["pid"] == 999999
|
||||
assert "not found" in body["reason"]
|
||||
|
||||
|
||||
def test_inspect_run_live_pid(client, monkeypatch):
|
||||
"""Run with a live PID returns alive=true with psutil fields."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="live-pid", assignee="heidi")
|
||||
run_id = _insert_run(conn, task_id, worker_pid=12345)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Build a realistic mock psutil.
|
||||
mock_psutil = MagicMock()
|
||||
mock_psutil.NoSuchProcess = type("NoSuchProcess", (Exception,), {})
|
||||
mock_psutil.AccessDenied = type("AccessDenied", (Exception,), {})
|
||||
|
||||
fake_mem = MagicMock()
|
||||
fake_mem.rss = 1024 * 1024 * 50 # 50 MB
|
||||
fake_mem.vms = 1024 * 1024 * 200
|
||||
|
||||
fake_proc = MagicMock()
|
||||
fake_proc.as_dict.return_value = {
|
||||
"cpu_percent": 3.5,
|
||||
"memory_info": fake_mem,
|
||||
"num_threads": 4,
|
||||
"status": "sleeping",
|
||||
"create_time": time.time() - 300,
|
||||
"cmdline": ["python", "-m", "hermes"],
|
||||
}
|
||||
fake_proc.num_fds.return_value = 12
|
||||
mock_psutil.Process.return_value = fake_proc
|
||||
|
||||
plugin_mod_name = "hermes_dashboard_plugin_kanban_worker_runs_test"
|
||||
plugin_mod = sys.modules.get(plugin_mod_name)
|
||||
if plugin_mod is not None:
|
||||
monkeypatch.setattr(plugin_mod, "_psutil", mock_psutil)
|
||||
else:
|
||||
pytest.skip("plugin module not yet loaded")
|
||||
|
||||
r = client.get(f"/api/plugins/kanban/runs/{run_id}/inspect")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["alive"] is True
|
||||
assert body["pid"] == 12345
|
||||
assert body["cpu_percent"] == 3.5
|
||||
assert body["memory_rss_bytes"] == fake_mem.rss
|
||||
assert body["num_threads"] == 4
|
||||
assert body["status"] == "sleeping"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# POST /runs/{run_id}/terminate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _setup_running_task_with_run(conn, *, title, assignee, worker_pid):
|
||||
"""Create a task in 'running' state with a matching open task_runs row.
|
||||
|
||||
Mirrors what dispatcher_claim does: stamps tasks.status='running',
|
||||
tasks.claim_lock, tasks.worker_pid; inserts task_runs row with the
|
||||
same claim_lock so reclaim_task's preconditions are satisfied.
|
||||
"""
|
||||
task_id = kb.create_task(conn, title=title, assignee=assignee)
|
||||
lock = secrets.token_hex(8)
|
||||
future = int(time.time()) + 3600
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status='running', claim_lock=?, "
|
||||
"claim_expires=?, worker_pid=? WHERE id=?",
|
||||
(lock, future, worker_pid, task_id),
|
||||
)
|
||||
cur = conn.execute(
|
||||
"INSERT INTO task_runs "
|
||||
"(task_id, status, claim_lock, claim_expires, worker_pid, started_at) "
|
||||
"VALUES (?, 'running', ?, ?, ?, ?)",
|
||||
(task_id, lock, future, worker_pid, int(time.time())),
|
||||
)
|
||||
conn.commit()
|
||||
return task_id, cur.lastrowid
|
||||
|
||||
|
||||
def test_terminate_run_404_unknown_id(client):
|
||||
"""POST to unknown run_id returns 404."""
|
||||
r = client.post(
|
||||
"/api/plugins/kanban/runs/777777/terminate",
|
||||
json={"reason": "test"},
|
||||
)
|
||||
assert r.status_code == 404
|
||||
assert "777777" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_terminate_run_409_already_ended(client):
|
||||
"""POST against a run with ended_at set returns 409."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="ended-terminate", assignee="ivy")
|
||||
run_id = _insert_run(
|
||||
conn, task_id, worker_pid=22222, ended_at=int(time.time()) - 30,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
r = client.post(
|
||||
f"/api/plugins/kanban/runs/{run_id}/terminate",
|
||||
json={"reason": "too late"},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert "already ended" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_terminate_run_ok(client, monkeypatch):
|
||||
"""Happy path: live run is terminated, signal fn invoked, reason recorded."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id, run_id = _setup_running_task_with_run(
|
||||
conn, title="kill-me", assignee="jane", worker_pid=33333,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Capture signal calls so we don't actually SIGTERM a random PID.
|
||||
sent = []
|
||||
|
||||
def _fake_terminate(pid, prev_lock, *, signal_fn=None):
|
||||
sent.append((pid, prev_lock))
|
||||
return {"signal": "SIGTERM", "delivered": True}
|
||||
|
||||
monkeypatch.setattr(kb, "_terminate_reclaimed_worker", _fake_terminate)
|
||||
|
||||
r = client.post(
|
||||
f"/api/plugins/kanban/runs/{run_id}/terminate",
|
||||
json={"reason": "operator abort"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body == {"ok": True, "run_id": run_id, "task_id": task_id}
|
||||
assert sent == [(33333, sent[0][1])]
|
||||
assert sent[0][1] is not None # claim_lock was non-null
|
||||
|
||||
# Task is back to ready, claim cleared.
|
||||
conn = kb.connect()
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT status, claim_lock, worker_pid FROM tasks WHERE id=?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
assert row["status"] == "ready"
|
||||
assert row["claim_lock"] is None
|
||||
assert row["worker_pid"] is None
|
||||
|
||||
|
||||
def test_terminate_run_409_task_not_reclaimable(client, monkeypatch):
|
||||
"""Open run row whose task is no longer claimable returns 409."""
|
||||
conn = kb.connect()
|
||||
try:
|
||||
task_id = kb.create_task(conn, title="ghost-run", assignee="ken")
|
||||
# Task left in default 'ready' state with no claim_lock — task_run
|
||||
# exists but reclaim_task will refuse because status != 'running'
|
||||
# and claim_lock is NULL.
|
||||
run_id = _insert_run(conn, task_id, worker_pid=44444)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Make sure no signal is ever sent on this code path.
|
||||
def _boom(*a, **k):
|
||||
raise AssertionError("_terminate_reclaimed_worker should not be called")
|
||||
|
||||
monkeypatch.setattr(kb, "_terminate_reclaimed_worker", _boom)
|
||||
|
||||
r = client.post(
|
||||
f"/api/plugins/kanban/runs/{run_id}/terminate",
|
||||
json={"reason": "stale"},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert "reclaimable" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_terminate_run_accepts_empty_body(client):
|
||||
"""Empty JSON body (no reason) is still accepted; falls through to 404."""
|
||||
r = client.post(
|
||||
"/api/plugins/kanban/runs/666666/terminate",
|
||||
json={},
|
||||
)
|
||||
# 404 because run doesn't exist — what we're asserting here is that
|
||||
# the endpoint doesn't 422 on a missing 'reason' field.
|
||||
assert r.status_code == 404
|
||||
@@ -0,0 +1,706 @@
|
||||
"""Tests for the bundled observability/langfuse plugin."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PLUGIN_DIR = REPO_ROOT / "plugins" / "observability" / "langfuse"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manifest + layout
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestManifest:
|
||||
def test_plugin_directory_exists(self):
|
||||
assert PLUGIN_DIR.is_dir()
|
||||
assert (PLUGIN_DIR / "plugin.yaml").exists()
|
||||
assert (PLUGIN_DIR / "__init__.py").exists()
|
||||
|
||||
def test_manifest_fields(self):
|
||||
data = yaml.safe_load((PLUGIN_DIR / "plugin.yaml").read_text())
|
||||
assert data["name"] == "langfuse"
|
||||
assert data["version"]
|
||||
# All six hooks the plugin implements.
|
||||
assert set(data["hooks"]) == {
|
||||
"pre_api_request", "post_api_request",
|
||||
"pre_llm_call", "post_llm_call",
|
||||
"pre_tool_call", "post_tool_call",
|
||||
}
|
||||
# Required env vars are the user-facing HERMES_ prefixed keys.
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in data["requires_env"]
|
||||
assert "HERMES_LANGFUSE_SECRET_KEY" in data["requires_env"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin discovery: langfuse is opt-in (not loaded unless explicitly enabled).
|
||||
# This guards against someone accidentally re-introducing a per-hook
|
||||
# load_config() gate or making the plugin auto-load.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDiscovery:
|
||||
def test_plugin_is_discovered_as_standalone_opt_in(self, tmp_path, monkeypatch):
|
||||
"""Scanner should find the plugin but NOT load it by default."""
|
||||
from hermes_cli import plugins as plugins_mod
|
||||
|
||||
# Isolated HERMES_HOME so we don't read the developer's config.yaml.
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
|
||||
manager = plugins_mod.PluginManager()
|
||||
manager.discover_and_load()
|
||||
|
||||
# observability/langfuse appears in the plugin registry …
|
||||
loaded = manager._plugins.get("observability/langfuse")
|
||||
assert loaded is not None, "plugin not discovered"
|
||||
# … but is not loaded (opt-in default → no config.yaml means nothing enabled)
|
||||
assert loaded.enabled is False
|
||||
assert "not enabled" in (loaded.error or "").lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runtime gate: _get_langfuse() returns None and caches _INIT_FAILED when
|
||||
# credentials are missing. Guards against regressing toward the rejected
|
||||
# per-hook load_config() design.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRuntimeGate:
|
||||
def _fresh_plugin(self):
|
||||
"""Import the plugin module fresh (clears any cached client)."""
|
||||
mod_name = "plugins.observability.langfuse"
|
||||
sys.modules.pop(mod_name, None)
|
||||
return importlib.import_module(mod_name)
|
||||
|
||||
def test_get_langfuse_returns_none_without_credentials(self, monkeypatch):
|
||||
for k in (
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
langfuse_plugin = self._fresh_plugin()
|
||||
assert langfuse_plugin._get_langfuse() is None
|
||||
|
||||
def test_get_langfuse_caches_failure_no_config_load(self, monkeypatch):
|
||||
"""A miss must be cached — no per-hook config.yaml reads, no env re-reads."""
|
||||
for k in (
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
langfuse_plugin = self._fresh_plugin()
|
||||
|
||||
# Prime the cache with one call.
|
||||
assert langfuse_plugin._get_langfuse() is None
|
||||
|
||||
# Now block os.environ.get — a correctly-cached plugin must not
|
||||
# touch env again.
|
||||
import os
|
||||
called = {"n": 0}
|
||||
real_get = os.environ.get
|
||||
|
||||
def tracking_get(key, default=None):
|
||||
if key.startswith(("HERMES_LANGFUSE_", "LANGFUSE_")):
|
||||
called["n"] += 1
|
||||
return real_get(key, default)
|
||||
|
||||
monkeypatch.setattr(os.environ, "get", tracking_get)
|
||||
|
||||
for _ in range(20):
|
||||
assert langfuse_plugin._get_langfuse() is None
|
||||
|
||||
assert called["n"] == 0, (
|
||||
f"_get_langfuse() re-read env {called['n']} times after cache miss — "
|
||||
"it should short-circuit via _INIT_FAILED"
|
||||
)
|
||||
|
||||
def test_get_langfuse_does_not_import_hermes_config(self, monkeypatch):
|
||||
"""The plugin must not re-read config.yaml per hook."""
|
||||
for k in (
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
# Drop any cached import of hermes_cli.config.
|
||||
sys.modules.pop("hermes_cli.config", None)
|
||||
|
||||
langfuse_plugin = self._fresh_plugin()
|
||||
for _ in range(20):
|
||||
langfuse_plugin._get_langfuse()
|
||||
|
||||
assert "hermes_cli.config" not in sys.modules, (
|
||||
"langfuse plugin imported hermes_cli.config — regression toward "
|
||||
"the rejected per-hook load_config() design"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hooks are inert when the client is unavailable.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestHooksInert:
|
||||
def test_hooks_noop_without_client(self, monkeypatch):
|
||||
"""All 6 hooks must return without raising when _get_langfuse() is None."""
|
||||
for k in (
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
import importlib
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
# Each hook should just return; no exceptions.
|
||||
mod.on_pre_llm_call(task_id="t", session_id="s", messages=[{"role": "user", "content": "hi"}])
|
||||
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, request_messages=[])
|
||||
mod.on_post_llm_call(task_id="t", session_id="s", api_call_count=1)
|
||||
mod.on_pre_tool_call(tool_name="read_file", args={}, task_id="t", session_id="s")
|
||||
mod.on_post_tool_call(tool_name="read_file", args={}, result="ok", task_id="t", session_id="s")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Placeholder-credential guard (#23823).
|
||||
#
|
||||
# Regression coverage for the silent-failure bug: when an operator leaves
|
||||
# HERMES_LANGFUSE_PUBLIC_KEY / SECRET_KEY at a template value like
|
||||
# "placeholder", "test-key", or "your-langfuse-key", the SDK accepts the
|
||||
# credentials at construction time (it does no server-side validation
|
||||
# eagerly) but drops every trace at flush time, with no signal in the
|
||||
# Hermes logs. The fix in `_get_langfuse()` validates the documented
|
||||
# `pk-lf-` / `sk-lf-` prefix Langfuse always issues, surfaces a one-shot
|
||||
# warning naming the offending env var(s), and short-circuits via the
|
||||
# same `_INIT_FAILED` path used for missing credentials so subsequent
|
||||
# hook invocations don't re-log.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeLangfuse:
|
||||
"""Stand-in for the real :class:`langfuse.Langfuse` so tests don't
|
||||
need the optional ``langfuse`` SDK installed. The plugin's runtime
|
||||
gate refuses to proceed past ``if Langfuse is None`` when the SDK
|
||||
is missing, which would short-circuit before the placeholder check
|
||||
can fire. Patching ``plugin.Langfuse`` with this class lets the
|
||||
placeholder validator exercise its full code path."""
|
||||
|
||||
instances: list["_FakeLangfuse"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
_FakeLangfuse.instances.append(self)
|
||||
|
||||
|
||||
class TestPlaceholderKeyDetection:
|
||||
LOGGER_NAME = "plugins.observability.langfuse"
|
||||
|
||||
def _fresh_plugin(self, monkeypatch=None):
|
||||
mod_name = "plugins.observability.langfuse"
|
||||
sys.modules.pop(mod_name, None)
|
||||
mod = importlib.import_module(mod_name)
|
||||
if monkeypatch is not None:
|
||||
# Pretend the SDK is installed so `_get_langfuse()` actually
|
||||
# reaches the placeholder check. Real SDK calls are never
|
||||
# made because the placeholder/missing-credentials paths
|
||||
# return before constructing a client.
|
||||
_FakeLangfuse.instances.clear()
|
||||
monkeypatch.setattr(mod, "Langfuse", _FakeLangfuse, raising=False)
|
||||
return mod
|
||||
|
||||
@staticmethod
|
||||
def _clear_env(monkeypatch):
|
||||
for k in (
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
# -- helper unit tests (no SDK stub needed: these don't go through
|
||||
# _get_langfuse, they exercise the pure-Python helpers directly) ------
|
||||
|
||||
def test_redact_key_preview_empty(self, monkeypatch):
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._redact_key_preview("") == "<empty>"
|
||||
|
||||
def test_redact_key_preview_short_value_echoed(self, monkeypatch):
|
||||
"""Short placeholder strings are echoed in full so the operator
|
||||
can see exactly which template they forgot to replace."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._redact_key_preview("placeholder") == "'placeholder'"
|
||||
assert plugin._redact_key_preview("test-key") == "'test-key'"
|
||||
|
||||
def test_redact_key_preview_long_value_truncated(self, monkeypatch):
|
||||
"""If an operator pasted a real secret into the wrong env var the
|
||||
preview must NOT echo it in full — only the leading 6 chars."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
result = plugin._redact_key_preview("sk-lf-abcdefghijklmnop")
|
||||
assert "abcdefghij" not in result
|
||||
assert result.startswith("'sk-lf-")
|
||||
assert result.endswith("...'")
|
||||
|
||||
def test_validate_langfuse_key_accepts_documented_prefix(self, monkeypatch):
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._validate_langfuse_key(
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz"
|
||||
) is None
|
||||
assert plugin._validate_langfuse_key(
|
||||
"HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz"
|
||||
) is None
|
||||
|
||||
def test_validate_langfuse_key_rejects_wrong_prefix(self, monkeypatch):
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
msg = plugin._validate_langfuse_key(
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "placeholder"
|
||||
)
|
||||
assert msg is not None
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in msg
|
||||
assert "pk-lf-" in msg
|
||||
|
||||
def test_validate_langfuse_key_unknown_name_passes(self, monkeypatch):
|
||||
"""Defensive: an env var with no registered prefix is trusted."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._validate_langfuse_key("HERMES_LANGFUSE_BASE_URL", "anything") is None
|
||||
|
||||
# -- end-to-end _get_langfuse() behaviour --------------------------------
|
||||
# These tests pass `monkeypatch` to _fresh_plugin() so the helper can
|
||||
# stub out `Langfuse` (the optional SDK). Without that, every call
|
||||
# short-circuits at `if Langfuse is None` before reaching the
|
||||
# placeholder validator — masking the very behaviour we're testing.
|
||||
|
||||
def test_placeholder_public_key_warns_and_skips(self, monkeypatch, caplog):
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
text = caplog.text
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
|
||||
assert "'placeholder'" in text
|
||||
assert "pk-lf-" in text
|
||||
# The valid secret value must NOT appear (the var NAME does, in
|
||||
# the "or unset ..." hint, but the value preview shouldn't).
|
||||
assert "'sk-lf-" not in text
|
||||
# Never constructed the SDK client — short-circuited before that.
|
||||
assert _FakeLangfuse.instances == []
|
||||
|
||||
def test_placeholder_secret_key_warns_and_skips(self, monkeypatch, caplog):
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "test-key")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
text = caplog.text
|
||||
assert "HERMES_LANGFUSE_SECRET_KEY" in text
|
||||
assert "'test-key'" in text
|
||||
assert "sk-lf-" in text
|
||||
# The valid public value must NOT appear.
|
||||
assert "'pk-lf-" not in text
|
||||
assert _FakeLangfuse.instances == []
|
||||
|
||||
def test_both_placeholders_one_warning_with_both_keys(self, monkeypatch, caplog):
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert len(warnings) == 1, (
|
||||
f"Expected a single combined warning; got {len(warnings)}:\n"
|
||||
+ "\n".join(r.getMessage() for r in warnings)
|
||||
)
|
||||
text = warnings[0].getMessage()
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
|
||||
assert "HERMES_LANGFUSE_SECRET_KEY" in text
|
||||
|
||||
def test_repeated_calls_do_not_re_warn(self, monkeypatch, caplog):
|
||||
"""The cached ``_INIT_FAILED`` sentinel must short-circuit
|
||||
subsequent calls so each hook invocation isn't a fresh log
|
||||
line — otherwise a busy gateway will spam the operator's
|
||||
terminal."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
for _ in range(15):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert len(warnings) == 1, (
|
||||
f"Warning fired {len(warnings)} times across 15 calls; "
|
||||
"expected 1 (cached via _INIT_FAILED)"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("placeholder", [
|
||||
"placeholder",
|
||||
"test-key",
|
||||
"your-langfuse-key",
|
||||
"change-me",
|
||||
"xxx",
|
||||
"dummy-key-here",
|
||||
"<your-key>",
|
||||
"REPLACE_ME",
|
||||
])
|
||||
def test_common_placeholders_detected(self, monkeypatch, caplog, placeholder):
|
||||
"""A grab-bag of values that real-world ``.env.example`` templates
|
||||
use as stand-ins. Any of them in either key must trip the guard."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", placeholder)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
|
||||
|
||||
def test_legacy_LANGFUSE_PUBLIC_KEY_also_validated(self, monkeypatch, caplog):
|
||||
"""The plugin reads both the canonical HERMES_-prefixed env var and
|
||||
the legacy bare ``LANGFUSE_PUBLIC_KEY``. The validator must run on
|
||||
whichever value ``_get_langfuse()`` actually consumed."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
# Warning names the canonical user-facing env var (the bare
|
||||
# LANGFUSE_PUBLIC_KEY is a backwards-compat alias for the
|
||||
# HERMES_-prefixed one — operators set the HERMES_-prefixed one).
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
|
||||
assert "'placeholder'" in caplog.text
|
||||
|
||||
def test_missing_credentials_still_skip_silently(self, monkeypatch, caplog):
|
||||
"""Missing-creds is the documented opt-out path (operator hasn't
|
||||
configured the plugin yet) — it must remain SILENT. Regression
|
||||
guard against the placeholder validator accidentally running on
|
||||
empty values and re-introducing log noise for unconfigured
|
||||
installs."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert warnings == []
|
||||
|
||||
def test_sdk_not_installed_still_skips_silently(self, monkeypatch, caplog):
|
||||
"""If the langfuse SDK isn't installed at all, the placeholder
|
||||
check should never run — there's nothing the operator can do
|
||||
about a credential mismatch when the package is missing, and
|
||||
re-warning here would dilute the actually-actionable SDK-missing
|
||||
signal upstream. The ``Langfuse is None`` guard at the top of
|
||||
``_get_langfuse`` already handles this; this test pins that
|
||||
behaviour."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
|
||||
# NO monkeypatch on Langfuse here — falls back to whatever the
|
||||
# plugin imported at module load (None if SDK absent).
|
||||
plugin = self._fresh_plugin()
|
||||
monkeypatch.setattr(plugin, "Langfuse", None, raising=False)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert warnings == []
|
||||
|
||||
def test_valid_prefixes_do_not_trigger_placeholder_warning(self, monkeypatch, caplog):
|
||||
"""Real Langfuse keys (``pk-lf-…`` / ``sk-lf-…``) must pass the
|
||||
guard and proceed to SDK init. We stub the SDK constructor with
|
||||
a recording fake so the assertion can confirm BOTH that the
|
||||
placeholder warning didn't fire AND that the client was actually
|
||||
constructed — the latter is the success signal the bug report
|
||||
wanted."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
client = plugin._get_langfuse()
|
||||
assert isinstance(client, _FakeLangfuse)
|
||||
assert client.kwargs["public_key"] == "pk-lf-real-public-xyz"
|
||||
assert client.kwargs["secret_key"] == "sk-lf-real-secret-xyz"
|
||||
assert "placeholders" not in caplog.text.lower(), (
|
||||
f"Valid Langfuse keys tripped the placeholder guard: {caplog.text!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestRequestMessageCoercion:
|
||||
def test_prefers_request_messages_then_messages_then_history_then_user_message(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
assert mod._coerce_request_messages(
|
||||
request_messages=[{"role": "system", "content": "s"}],
|
||||
messages=[{"role": "user", "content": "m"}],
|
||||
conversation_history=[{"role": "user", "content": "h"}],
|
||||
user_message="u",
|
||||
) == [{"role": "system", "content": "s"}]
|
||||
assert mod._coerce_request_messages(
|
||||
messages=[{"role": "user", "content": "m"}],
|
||||
conversation_history=[{"role": "user", "content": "h"}],
|
||||
user_message="u",
|
||||
) == [{"role": "user", "content": "m"}]
|
||||
assert mod._coerce_request_messages(
|
||||
conversation_history=[{"role": "user", "content": "h"}],
|
||||
user_message="u",
|
||||
) == [{"role": "user", "content": "h"}]
|
||||
assert mod._coerce_request_messages(user_message="u") == [{"role": "user", "content": "u"}]
|
||||
|
||||
|
||||
class TestToolCallOutputBackfill:
|
||||
def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
observation = object()
|
||||
state = mod.TraceState(trace_id="trace-1", root_ctx=None, root_span=None)
|
||||
state.tools["call-1"] = observation
|
||||
state.turn_tool_calls.append({
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
"function": {
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
},
|
||||
})
|
||||
|
||||
task_key = mod._trace_key("task-1", "session-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
ended = {}
|
||||
|
||||
def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):
|
||||
ended["observation"] = obs
|
||||
ended["output"] = output
|
||||
ended["metadata"] = metadata
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end_observation)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract",
|
||||
args={"urls": ["https://example.com"]},
|
||||
result='{"results": [{"url": "https://example.com", "content": "Example Domain"}]}',
|
||||
task_id="task-1",
|
||||
session_id="session-1",
|
||||
tool_call_id="call-1",
|
||||
)
|
||||
|
||||
assert ended["observation"] is observation
|
||||
assert state.turn_tool_calls[0]["output"] == ended["output"]
|
||||
assert state.turn_tool_calls[0]["function"]["output"] == ended["output"]
|
||||
assert state.turn_tool_calls[0]["output"] == {
|
||||
"results": [{"url": "https://example.com", "content": "Example Domain"}]
|
||||
}
|
||||
|
||||
def test_serialize_messages_keeps_tool_name_and_call_id(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
messages = [{
|
||||
"role": "tool",
|
||||
"name": "web_extract",
|
||||
"tool_call_id": "call-1",
|
||||
"content": '{"ok": true}',
|
||||
}]
|
||||
|
||||
assert mod._serialize_messages(messages) == [{
|
||||
"role": "tool",
|
||||
"name": "web_extract",
|
||||
"tool_call_id": "call-1",
|
||||
"content": {"ok": True},
|
||||
}]
|
||||
|
||||
def test_serialize_tool_calls_emits_openai_style_function_shape(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
class _Fn:
|
||||
name = "web_extract"
|
||||
arguments = '{"urls": ["https://example.com"]}'
|
||||
|
||||
class _ToolCall:
|
||||
id = "call-1"
|
||||
type = "function"
|
||||
function = _Fn()
|
||||
|
||||
assert mod._serialize_tool_calls([_ToolCall()]) == [{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
"function": {
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
},
|
||||
}]
|
||||
|
||||
|
||||
class TestToolObservationKeying:
|
||||
"""Tests for pre/post tool_call observation matching when tool_call_id is absent."""
|
||||
|
||||
def _make_mod(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
return importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
def test_empty_tool_call_id_single_tool_sets_output(self, monkeypatch):
|
||||
mod = self._make_mod()
|
||||
obs = object()
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.pending_tools_by_name.setdefault("my_tool", []).append(obs)
|
||||
|
||||
task_key = mod._trace_key("task-1", "sess-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
ended = {}
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
ended["obs"] = o
|
||||
ended["output"] = output
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="my_tool",
|
||||
args={},
|
||||
result='{"ok": true}',
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
tool_call_id="",
|
||||
)
|
||||
|
||||
assert ended["obs"] is obs
|
||||
assert ended["output"] == {"ok": True}
|
||||
assert state.pending_tools_by_name.get("my_tool") is None
|
||||
|
||||
def test_empty_tool_call_id_observations_are_fifo_within_tool_name(self, monkeypatch):
|
||||
"""Two queued observations are consumed in FIFO order so the first
|
||||
post hook gets the first observation's output, not the second.
|
||||
|
||||
Sequential-on-one-thread coverage; the real concurrent case is
|
||||
guarded by ``_STATE_LOCK`` around every read-modify-write on
|
||||
``pending_tools_by_name`` and is exercised in
|
||||
``test_threaded_post_calls_preserve_fifo_under_lock`` below.
|
||||
"""
|
||||
mod = self._make_mod()
|
||||
obs_a, obs_b = object(), object()
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.pending_tools_by_name["web_extract"] = [obs_a, obs_b]
|
||||
|
||||
task_key = mod._trace_key("task-1", "sess-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
calls.append((o, output))
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract", args={}, result='{"val": "a"}',
|
||||
task_id="task-1", session_id="sess-1", tool_call_id="",
|
||||
)
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract", args={}, result='{"val": "b"}',
|
||||
task_id="task-1", session_id="sess-1", tool_call_id="",
|
||||
)
|
||||
|
||||
assert calls[0] == (obs_a, {"val": "a"})
|
||||
assert calls[1] == (obs_b, {"val": "b"})
|
||||
assert state.pending_tools_by_name.get("web_extract") is None
|
||||
|
||||
def test_threaded_post_calls_preserve_fifo_under_lock(self, monkeypatch):
|
||||
"""The actual concurrency contract: when 8 threads race to drain
|
||||
the pending queue, no observation is consumed twice and none is
|
||||
lost. Validates ``_STATE_LOCK`` discipline, not Python list
|
||||
semantics."""
|
||||
import threading
|
||||
|
||||
mod = self._make_mod()
|
||||
n = 8
|
||||
observations = [object() for _ in range(n)]
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.pending_tools_by_name["web_extract"] = list(observations)
|
||||
|
||||
task_key = mod._trace_key("task-thr", "sess-thr")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
recorded: list = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
with lock:
|
||||
recorded.append(o)
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
barrier = threading.Barrier(n)
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract", args={}, result='{"ok": true}',
|
||||
task_id="task-thr", session_id="sess-thr", tool_call_id="",
|
||||
)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(n)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Every observation was consumed exactly once; queue is empty.
|
||||
assert len(recorded) == n
|
||||
assert set(map(id, recorded)) == set(map(id, observations))
|
||||
assert state.pending_tools_by_name.get("web_extract") is None
|
||||
|
||||
def test_explicit_tool_call_id_uses_tools_dict(self, monkeypatch):
|
||||
"""When tool_call_id is present, pending_tools_by_name is not touched."""
|
||||
mod = self._make_mod()
|
||||
obs = object()
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.tools["call-99"] = obs
|
||||
|
||||
task_key = mod._trace_key("task-1", "sess-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
ended = {}
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
ended["obs"] = o
|
||||
ended["output"] = output
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="my_tool", args={}, result='{"status": "done"}',
|
||||
task_id="task-1", session_id="sess-1", tool_call_id="call-99",
|
||||
)
|
||||
|
||||
assert ended["obs"] is obs
|
||||
assert ended["output"] == {"status": "done"}
|
||||
assert not state.tools
|
||||
|
||||
@@ -0,0 +1,738 @@
|
||||
"""Tests for the RetainDB memory plugin.
|
||||
|
||||
Covers: _Client HTTP client, _WriteQueue SQLite queue, _build_overlay formatter,
|
||||
RetainDBMemoryProvider lifecycle/tools/prefetch, thread management, connection pooling.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Imports — guarded since plugins/memory lives outside the standard test path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(tmp_path, monkeypatch):
|
||||
"""Ensure HERMES_HOME and RETAINDB vars are isolated."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.delenv("RETAINDB_API_KEY", raising=False)
|
||||
monkeypatch.delenv("RETAINDB_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("RETAINDB_PROJECT", raising=False)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _cap_retaindb_sleeps(monkeypatch):
|
||||
"""Cap production-code sleeps so background-thread tests run fast.
|
||||
|
||||
The retaindb ``_WriteQueue._flush_row`` does ``time.sleep(2)`` after
|
||||
errors. Across multiple tests that trigger the retry path, that adds
|
||||
up. Cap the module's bound ``time.sleep`` to 0.05s — tests don't care
|
||||
about the exact retry delay, only that it happens. The test file's
|
||||
own ``time.sleep`` stays real since it uses a different reference.
|
||||
"""
|
||||
try:
|
||||
from plugins.memory import retaindb as _retaindb
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
real_sleep = _retaindb.time.sleep
|
||||
|
||||
def _capped_sleep(seconds):
|
||||
return real_sleep(min(float(seconds), 0.05))
|
||||
|
||||
import types as _types
|
||||
fake_time = _types.SimpleNamespace(sleep=_capped_sleep, time=_retaindb.time.time)
|
||||
monkeypatch.setattr(_retaindb, "time", fake_time)
|
||||
|
||||
|
||||
# We need the repo root on sys.path so the plugin can import agent.memory_provider
|
||||
import sys
|
||||
_repo_root = str(Path(__file__).resolve().parents[2])
|
||||
if _repo_root not in sys.path:
|
||||
sys.path.insert(0, _repo_root)
|
||||
|
||||
from plugins.memory.retaindb import (
|
||||
_Client,
|
||||
_WriteQueue,
|
||||
_build_overlay,
|
||||
RetainDBMemoryProvider,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _Client tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestClient:
|
||||
"""Test the HTTP client with mocked requests."""
|
||||
|
||||
def _make_client(self, api_key="rdb-test-key", base_url="https://api.retaindb.com", project="test"):
|
||||
return _Client(api_key, base_url, project)
|
||||
|
||||
def test_base_url_trailing_slash_stripped(self):
|
||||
c = self._make_client(base_url="https://api.retaindb.com///")
|
||||
assert c.base_url == "https://api.retaindb.com"
|
||||
|
||||
def test_headers_include_auth(self):
|
||||
c = self._make_client()
|
||||
h = c._headers("/v1/files")
|
||||
assert h["Authorization"] == "Bearer rdb-test-key"
|
||||
assert "X-API-Key" not in h
|
||||
|
||||
def test_headers_include_api_key_for_memory_path(self):
|
||||
c = self._make_client()
|
||||
h = c._headers("/v1/memory/search")
|
||||
assert h["X-API-Key"] == "rdb-test-key"
|
||||
|
||||
def test_headers_include_api_key_for_context_path(self):
|
||||
c = self._make_client()
|
||||
h = c._headers("/v1/context/query")
|
||||
assert h["X-API-Key"] == "rdb-test-key"
|
||||
|
||||
def test_headers_strip_bearer_prefix(self):
|
||||
c = self._make_client(api_key="Bearer rdb-test-key")
|
||||
h = c._headers("/v1/memory/search")
|
||||
assert h["Authorization"] == "Bearer rdb-test-key"
|
||||
assert h["X-API-Key"] == "rdb-test-key"
|
||||
|
||||
def test_add_memory_tries_fallback(self):
|
||||
c = self._make_client()
|
||||
call_count = 0
|
||||
def fake_request(method, path, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
raise RuntimeError("404")
|
||||
return {"id": "mem-1"}
|
||||
|
||||
with patch.object(c, "request", side_effect=fake_request):
|
||||
result = c.add_memory("u1", "s1", "test fact")
|
||||
assert result == {"id": "mem-1"}
|
||||
assert call_count == 2
|
||||
|
||||
def test_delete_memory_tries_fallback(self):
|
||||
c = self._make_client()
|
||||
call_count = 0
|
||||
def fake_request(method, path, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
raise RuntimeError("404")
|
||||
return {"deleted": True}
|
||||
|
||||
with patch.object(c, "request", side_effect=fake_request):
|
||||
result = c.delete_memory("mem-123")
|
||||
assert result == {"deleted": True}
|
||||
assert call_count == 2
|
||||
|
||||
# ===========================================================================
|
||||
# _WriteQueue tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestWriteQueue:
|
||||
"""Test the SQLite-backed write queue with real SQLite."""
|
||||
|
||||
def _make_queue(self, tmp_path, client=None):
|
||||
if client is None:
|
||||
client = MagicMock()
|
||||
client.ingest_session = MagicMock(return_value={"status": "ok"})
|
||||
db_path = tmp_path / "test_queue.db"
|
||||
return _WriteQueue(client, db_path), client, db_path
|
||||
|
||||
def test_enqueue_creates_row(self, tmp_path):
|
||||
q, client, db_path = self._make_queue(tmp_path)
|
||||
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
|
||||
# shutdown() blocks until the writer thread drains the queue — no need
|
||||
# to pre-sleep (the old 1s sleep was a just-in-case wait, but shutdown
|
||||
# does the right thing).
|
||||
q.shutdown()
|
||||
# If ingest succeeded, the row should be deleted
|
||||
client.ingest_session.assert_called_once()
|
||||
|
||||
def test_enqueue_persists_to_sqlite(self, tmp_path):
|
||||
client = MagicMock()
|
||||
# Make ingest slow so the row is still in SQLite when we peek.
|
||||
# 0.5s is plenty — the test just needs the flush to still be in-flight.
|
||||
client.ingest_session = MagicMock(side_effect=lambda *a, **kw: time.sleep(0.5))
|
||||
db_path = tmp_path / "test_queue.db"
|
||||
q = _WriteQueue(client, db_path)
|
||||
q.enqueue("user1", "sess1", [{"role": "user", "content": "test"}])
|
||||
# Check SQLite directly — row should exist since flush is slow
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
rows = conn.execute("SELECT user_id, session_id FROM pending").fetchall()
|
||||
conn.close()
|
||||
assert len(rows) >= 1
|
||||
assert rows[0][0] == "user1"
|
||||
q.shutdown()
|
||||
|
||||
def test_flush_deletes_row_on_success(self, tmp_path):
|
||||
q, client, db_path = self._make_queue(tmp_path)
|
||||
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
|
||||
q.shutdown() # blocks until drain
|
||||
# Row should be gone
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
rows = conn.execute("SELECT COUNT(*) FROM pending").fetchone()[0]
|
||||
conn.close()
|
||||
assert rows == 0
|
||||
|
||||
def test_flush_records_error_on_failure(self, tmp_path):
|
||||
client = MagicMock()
|
||||
client.ingest_session = MagicMock(side_effect=RuntimeError("API down"))
|
||||
db_path = tmp_path / "test_queue.db"
|
||||
q = _WriteQueue(client, db_path)
|
||||
q.enqueue("user1", "sess1", [{"role": "user", "content": "hi"}])
|
||||
# Poll for the error to be recorded (max 2s), instead of a fixed 3s wait.
|
||||
deadline = time.time() + 2.0
|
||||
last_error = None
|
||||
while time.time() < deadline:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
row = conn.execute("SELECT last_error FROM pending").fetchone()
|
||||
conn.close()
|
||||
if row and row[0]:
|
||||
last_error = row[0]
|
||||
break
|
||||
time.sleep(0.05)
|
||||
q.shutdown()
|
||||
assert last_error is not None
|
||||
assert "API down" in last_error
|
||||
|
||||
def test_thread_local_connection_reuse(self, tmp_path):
|
||||
q, _, _ = self._make_queue(tmp_path)
|
||||
# Same thread should get same connection
|
||||
conn1 = q._get_conn()
|
||||
conn2 = q._get_conn()
|
||||
assert conn1 is conn2
|
||||
q.shutdown()
|
||||
|
||||
def test_crash_recovery_replays_pending(self, tmp_path):
|
||||
"""Simulate crash: create rows, then new queue should replay them."""
|
||||
db_path = tmp_path / "recovery_test.db"
|
||||
# First: create a queue and insert rows, but don't let them flush
|
||||
client1 = MagicMock()
|
||||
client1.ingest_session = MagicMock(side_effect=RuntimeError("fail"))
|
||||
q1 = _WriteQueue(client1, db_path)
|
||||
q1.enqueue("user1", "sess1", [{"role": "user", "content": "lost turn"}])
|
||||
# Wait until the error is recorded (poll with short interval).
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
row = conn.execute("SELECT last_error FROM pending").fetchone()
|
||||
conn.close()
|
||||
if row and row[0]:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
q1.shutdown()
|
||||
|
||||
# Now create a new queue — it should replay the pending rows
|
||||
client2 = MagicMock()
|
||||
client2.ingest_session = MagicMock(return_value={"status": "ok"})
|
||||
q2 = _WriteQueue(client2, db_path)
|
||||
# Poll for the replay to happen.
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
if client2.ingest_session.called:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
q2.shutdown()
|
||||
|
||||
# The replayed row should have been ingested via client2
|
||||
client2.ingest_session.assert_called_once()
|
||||
call_args = client2.ingest_session.call_args
|
||||
assert call_args[0][0] == "user1" # user_id
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _build_overlay tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestBuildOverlay:
|
||||
"""Test the overlay formatter (pure function)."""
|
||||
|
||||
def test_empty_inputs_returns_empty(self):
|
||||
assert _build_overlay({}, {}) == ""
|
||||
|
||||
def test_empty_memories_returns_empty(self):
|
||||
assert _build_overlay({"memories": []}, {"results": []}) == ""
|
||||
|
||||
def test_profile_items_included(self):
|
||||
profile = {"memories": [{"content": "User likes Python"}]}
|
||||
result = _build_overlay(profile, {})
|
||||
assert "User likes Python" in result
|
||||
assert "[RetainDB Context]" in result
|
||||
|
||||
def test_query_results_included(self):
|
||||
query_result = {"results": [{"content": "Previous discussion about Rust"}]}
|
||||
result = _build_overlay({}, query_result)
|
||||
assert "Previous discussion about Rust" in result
|
||||
|
||||
def test_deduplication_removes_duplicates(self):
|
||||
profile = {"memories": [{"content": "User likes Python"}]}
|
||||
query_result = {"results": [{"content": "User likes Python"}]}
|
||||
result = _build_overlay(profile, query_result)
|
||||
assert result.count("User likes Python") == 1
|
||||
|
||||
def test_local_entries_filter(self):
|
||||
profile = {"memories": [{"content": "Already known fact"}]}
|
||||
result = _build_overlay(profile, {}, local_entries=["Already known fact"])
|
||||
# The profile item matches a local entry, should be filtered
|
||||
assert result == ""
|
||||
|
||||
def test_max_five_items_per_section(self):
|
||||
profile = {"memories": [{"content": f"Fact {i}"} for i in range(10)]}
|
||||
result = _build_overlay(profile, {})
|
||||
# Should only include first 5
|
||||
assert "Fact 0" in result
|
||||
assert "Fact 4" in result
|
||||
assert "Fact 5" not in result
|
||||
|
||||
def test_none_content_handled(self):
|
||||
profile = {"memories": [{"content": None}, {"content": "Real fact"}]}
|
||||
result = _build_overlay(profile, {})
|
||||
assert "Real fact" in result
|
||||
|
||||
def test_truncation_at_320_chars(self):
|
||||
long_content = "x" * 500
|
||||
profile = {"memories": [{"content": long_content}]}
|
||||
result = _build_overlay(profile, {})
|
||||
# Each item is compacted to 320 chars max
|
||||
for line in result.split("\n"):
|
||||
if line.startswith("- "):
|
||||
assert len(line) <= 322 # "- " + 320
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# RetainDBMemoryProvider tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestRetainDBMemoryProvider:
|
||||
"""Test the main plugin class."""
|
||||
|
||||
def _make_provider(self, tmp_path, monkeypatch, api_key="rdb-test-key"):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", api_key)
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
(tmp_path / ".hermes").mkdir(exist_ok=True)
|
||||
provider = RetainDBMemoryProvider()
|
||||
return provider
|
||||
|
||||
def test_name(self):
|
||||
p = RetainDBMemoryProvider()
|
||||
assert p.name == "retaindb"
|
||||
|
||||
def test_is_available_without_key(self):
|
||||
p = RetainDBMemoryProvider()
|
||||
assert p.is_available() is False
|
||||
|
||||
def test_is_available_with_key(self, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test")
|
||||
p = RetainDBMemoryProvider()
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_config_schema(self):
|
||||
p = RetainDBMemoryProvider()
|
||||
schema = p.get_config_schema()
|
||||
assert len(schema) == 3
|
||||
keys = [s["key"] for s in schema]
|
||||
assert "api_key" in keys
|
||||
assert "base_url" in keys
|
||||
assert "project" in keys
|
||||
|
||||
def test_initialize_creates_client_and_queue(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
assert p._client is not None
|
||||
assert p._queue is not None
|
||||
assert p._session_id == "test-session"
|
||||
p.shutdown()
|
||||
|
||||
def test_initialize_default_project(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
assert p._client.project == "default"
|
||||
p.shutdown()
|
||||
|
||||
def test_initialize_explicit_project(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_PROJECT", "my-project")
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
assert p._client.project == "my-project"
|
||||
p.shutdown()
|
||||
|
||||
def test_initialize_profile_project(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
profile_home = str(tmp_path / "profiles" / "coder")
|
||||
p.initialize("test-session", hermes_home=profile_home)
|
||||
assert p._client.project == "hermes-coder"
|
||||
p.shutdown()
|
||||
|
||||
def test_initialize_seeds_soul_md(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
soul_path = tmp_path / ".hermes" / "SOUL.md"
|
||||
soul_path.write_text("I am a helpful agent.")
|
||||
with patch.object(RetainDBMemoryProvider, "_seed_soul") as mock_seed:
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
# Give thread time to start
|
||||
time.sleep(0.5)
|
||||
mock_seed.assert_called_once_with("I am a helpful agent.")
|
||||
p.shutdown()
|
||||
|
||||
def test_system_prompt_block(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
block = p.system_prompt_block()
|
||||
assert "RetainDB Memory" in block
|
||||
assert "Active" in block
|
||||
p.shutdown()
|
||||
|
||||
def test_handle_tool_call_not_initialized(self):
|
||||
p = RetainDBMemoryProvider()
|
||||
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
|
||||
assert "error" in result
|
||||
assert "not initialized" in result["error"]
|
||||
|
||||
def test_handle_tool_call_unknown_tool(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_nonexistent", {}))
|
||||
assert result == {"error": "Unknown tool: retaindb_nonexistent"}
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_profile(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "get_profile", return_value={"memories": []}):
|
||||
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
|
||||
assert "memories" in result
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_search_requires_query(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_search", {}))
|
||||
assert result == {"error": "query is required"}
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_search(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "search", return_value={"results": [{"content": "found"}]}):
|
||||
result = json.loads(p.handle_tool_call("retaindb_search", {"query": "test"}))
|
||||
assert "results" in result
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_search_top_k_capped(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "search") as mock_search:
|
||||
mock_search.return_value = {"results": []}
|
||||
p.handle_tool_call("retaindb_search", {"query": "test", "top_k": 100})
|
||||
# top_k should be capped at 20
|
||||
assert mock_search.call_args[1]["top_k"] == 20
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_remember(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}):
|
||||
result = json.loads(p.handle_tool_call("retaindb_remember", {"content": "test fact"}))
|
||||
assert result["id"] == "mem-1"
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_remember_requires_content(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_remember", {}))
|
||||
assert result == {"error": "content is required"}
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_forget(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "delete_memory", return_value={"deleted": True}):
|
||||
result = json.loads(p.handle_tool_call("retaindb_forget", {"memory_id": "mem-1"}))
|
||||
assert result["deleted"] is True
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_forget_requires_id(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_forget", {}))
|
||||
assert result == {"error": "memory_id is required"}
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_context(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "query_context", return_value={"results": [{"content": "relevant"}]}), \
|
||||
patch.object(p._client, "get_profile", return_value={"memories": []}):
|
||||
result = json.loads(p.handle_tool_call("retaindb_context", {"query": "current task"}))
|
||||
assert "context" in result
|
||||
assert "raw" in result
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_file_list(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "list_files", return_value={"files": []}):
|
||||
result = json.loads(p.handle_tool_call("retaindb_list_files", {}))
|
||||
assert "files" in result
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_file_upload_missing_path(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_upload_file", {}))
|
||||
assert "error" in result
|
||||
|
||||
def test_dispatch_file_upload_not_found(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_upload_file", {"local_path": "/nonexistent/file.txt"}))
|
||||
assert "File not found" in result["error"]
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_file_read_requires_id(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_read_file", {}))
|
||||
assert result == {"error": "file_id is required"}
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_file_ingest_requires_id(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_ingest_file", {}))
|
||||
assert result == {"error": "file_id is required"}
|
||||
p.shutdown()
|
||||
|
||||
def test_dispatch_file_delete_requires_id(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
result = json.loads(p.handle_tool_call("retaindb_delete_file", {}))
|
||||
assert result == {"error": "file_id is required"}
|
||||
p.shutdown()
|
||||
|
||||
def test_handle_tool_call_wraps_exception(self, tmp_path, monkeypatch):
|
||||
p = self._make_provider(tmp_path, monkeypatch)
|
||||
p.initialize("test-session", hermes_home=str(tmp_path / ".hermes"))
|
||||
with patch.object(p._client, "get_profile", side_effect=RuntimeError("API exploded")):
|
||||
result = json.loads(p.handle_tool_call("retaindb_profile", {}))
|
||||
assert "API exploded" in result["error"]
|
||||
p.shutdown()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Prefetch and thread management tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestPrefetch:
|
||||
"""Test background prefetch and thread accumulation prevention."""
|
||||
|
||||
def _make_initialized_provider(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
return p
|
||||
|
||||
def test_queue_prefetch_skips_without_client(self):
|
||||
p = RetainDBMemoryProvider()
|
||||
p.queue_prefetch("test") # Should not raise
|
||||
|
||||
def test_prefetch_returns_empty_when_nothing_cached(self, tmp_path, monkeypatch):
|
||||
p = self._make_initialized_provider(tmp_path, monkeypatch)
|
||||
result = p.prefetch("test")
|
||||
assert result == ""
|
||||
p.shutdown()
|
||||
|
||||
def test_prefetch_consumes_context_result(self, tmp_path, monkeypatch):
|
||||
p = self._make_initialized_provider(tmp_path, monkeypatch)
|
||||
# Manually set the cached result
|
||||
with p._lock:
|
||||
p._context_result = "[RetainDB Context]\nProfile:\n- User likes tests"
|
||||
result = p.prefetch("test")
|
||||
assert "User likes tests" in result
|
||||
# Should be consumed
|
||||
assert p.prefetch("test") == ""
|
||||
p.shutdown()
|
||||
|
||||
def test_prefetch_consumes_dialectic_result(self, tmp_path, monkeypatch):
|
||||
p = self._make_initialized_provider(tmp_path, monkeypatch)
|
||||
with p._lock:
|
||||
p._dialectic_result = "User is a software engineer who prefers Python."
|
||||
result = p.prefetch("test")
|
||||
assert "[RetainDB User Synthesis]" in result
|
||||
assert "software engineer" in result
|
||||
p.shutdown()
|
||||
|
||||
def test_prefetch_consumes_agent_model(self, tmp_path, monkeypatch):
|
||||
p = self._make_initialized_provider(tmp_path, monkeypatch)
|
||||
with p._lock:
|
||||
p._agent_model = {
|
||||
"memory_count": 5,
|
||||
"persona": "Helpful coding assistant",
|
||||
"persistent_instructions": ["Be concise", "Use Python"],
|
||||
"working_style": "Direct and efficient",
|
||||
}
|
||||
result = p.prefetch("test")
|
||||
assert "[RetainDB Agent Self-Model]" in result
|
||||
assert "Helpful coding assistant" in result
|
||||
assert "Be concise" in result
|
||||
assert "Direct and efficient" in result
|
||||
p.shutdown()
|
||||
|
||||
def test_prefetch_skips_empty_agent_model(self, tmp_path, monkeypatch):
|
||||
p = self._make_initialized_provider(tmp_path, monkeypatch)
|
||||
with p._lock:
|
||||
p._agent_model = {"memory_count": 0}
|
||||
result = p.prefetch("test")
|
||||
assert "Agent Self-Model" not in result
|
||||
p.shutdown()
|
||||
|
||||
def test_thread_accumulation_guard(self, tmp_path, monkeypatch):
|
||||
"""Verify old prefetch threads are joined before new ones spawn."""
|
||||
p = self._make_initialized_provider(tmp_path, monkeypatch)
|
||||
# Mock the prefetch methods to be slow
|
||||
with patch.object(p, "_prefetch_context", side_effect=lambda q: time.sleep(0.5)), \
|
||||
patch.object(p, "_prefetch_dialectic", side_effect=lambda q: time.sleep(0.5)), \
|
||||
patch.object(p, "_prefetch_agent_model", side_effect=lambda: time.sleep(0.5)):
|
||||
p.queue_prefetch("query 1")
|
||||
first_threads = list(p._prefetch_threads)
|
||||
assert len(first_threads) == 3
|
||||
|
||||
# Call again — should join first batch before spawning new
|
||||
p.queue_prefetch("query 2")
|
||||
second_threads = list(p._prefetch_threads)
|
||||
assert len(second_threads) == 3
|
||||
# Should be different thread objects
|
||||
for t in second_threads:
|
||||
assert t not in first_threads
|
||||
p.shutdown()
|
||||
|
||||
def test_reasoning_level_short(self):
|
||||
assert RetainDBMemoryProvider._reasoning_level("hi") == "low"
|
||||
|
||||
def test_reasoning_level_medium(self):
|
||||
assert RetainDBMemoryProvider._reasoning_level("x" * 200) == "medium"
|
||||
|
||||
def test_reasoning_level_long(self):
|
||||
assert RetainDBMemoryProvider._reasoning_level("x" * 500) == "high"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# sync_turn tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestSyncTurn:
|
||||
"""Test turn synchronization via the write queue."""
|
||||
|
||||
def test_sync_turn_enqueues(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
with patch.object(p._queue, "enqueue") as mock_enqueue:
|
||||
p.sync_turn("user msg", "assistant msg")
|
||||
mock_enqueue.assert_called_once()
|
||||
args = mock_enqueue.call_args[0]
|
||||
assert args[0] == "default" # user_id
|
||||
assert args[1] == "test-session" # session_id
|
||||
msgs = args[2]
|
||||
assert len(msgs) == 2
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert msgs[1]["role"] == "assistant"
|
||||
p.shutdown()
|
||||
|
||||
def test_sync_turn_skips_empty_user_content(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
with patch.object(p._queue, "enqueue") as mock_enqueue:
|
||||
p.sync_turn("", "assistant msg")
|
||||
mock_enqueue.assert_not_called()
|
||||
p.shutdown()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# on_memory_write hook tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestOnMemoryWrite:
|
||||
"""Test the built-in memory mirror hook."""
|
||||
|
||||
def test_mirrors_add_action(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}) as mock_add:
|
||||
p.on_memory_write("add", "user", "User prefers dark mode")
|
||||
mock_add.assert_called_once()
|
||||
assert mock_add.call_args[1]["memory_type"] == "preference"
|
||||
p.shutdown()
|
||||
|
||||
def test_skips_non_add_action(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
with patch.object(p._client, "add_memory") as mock_add:
|
||||
p.on_memory_write("remove", "user", "something")
|
||||
mock_add.assert_not_called()
|
||||
p.shutdown()
|
||||
|
||||
def test_skips_empty_content(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
with patch.object(p._client, "add_memory") as mock_add:
|
||||
p.on_memory_write("add", "user", "")
|
||||
mock_add.assert_not_called()
|
||||
p.shutdown()
|
||||
|
||||
def test_memory_target_maps_to_type(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RETAINDB_API_KEY", "rdb-test-key")
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir(exist_ok=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
p = RetainDBMemoryProvider()
|
||||
p.initialize("test-session", hermes_home=str(hermes_home))
|
||||
with patch.object(p._client, "add_memory", return_value={"id": "mem-1"}) as mock_add:
|
||||
p.on_memory_write("add", "memory", "Some env fact")
|
||||
assert mock_add.call_args[1]["memory_type"] == "factual"
|
||||
p.shutdown()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# register() test
|
||||
# ===========================================================================
|
||||
|
||||
class TestRegister:
|
||||
def test_register_calls_register_memory_provider(self):
|
||||
from plugins.memory.retaindb import register
|
||||
ctx = MagicMock()
|
||||
register(ctx)
|
||||
ctx.register_memory_provider.assert_called_once()
|
||||
arg = ctx.register_memory_provider.call_args[0][0]
|
||||
assert isinstance(arg, RetainDBMemoryProvider)
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Tests for the security-guidance plugin.
|
||||
|
||||
Covers ``plugins/security-guidance/``:
|
||||
|
||||
* ``patterns.py`` data integrity — every rule has a ``RuleId``, the
|
||||
fail-loud import assertion is wired.
|
||||
* ``_scan_content`` — true positives (pickle.load, yaml.load, eval,
|
||||
dangerouslySetInnerHTML, GitHub Actions workflow), true negatives
|
||||
(.md skips Python rules, ``model.eval()`` doesn't trip eval),
|
||||
path-only rules (``path_check``), content-only rules
|
||||
(``path_filter``).
|
||||
* Hooks — ``transform_tool_result`` appends a warning block in warn
|
||||
mode and stays out of error results; ``pre_tool_call`` blocks
|
||||
writes when ``SECURITY_GUIDANCE_BLOCK=1`` and stays silent
|
||||
otherwise.
|
||||
* Bundled-plugin discovery via ``PluginManager.discover_and_load``.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.delenv("SECURITY_GUIDANCE_BLOCK", raising=False)
|
||||
monkeypatch.delenv("SECURITY_GUIDANCE_DISABLE", raising=False)
|
||||
yield hermes_home
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _load_patterns():
|
||||
"""Import patterns.py in isolation (no plugin glue)."""
|
||||
pat_path = _repo_root() / "plugins" / "security-guidance" / "patterns.py"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"security_guidance_patterns_under_test", pat_path
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def _load_plugin_init():
|
||||
"""Import the plugin __init__.py with patterns.py as a sibling."""
|
||||
plugin_dir = _repo_root() / "plugins" / "security-guidance"
|
||||
if "hermes_plugins" not in sys.modules:
|
||||
ns = types.ModuleType("hermes_plugins")
|
||||
ns.__path__ = []
|
||||
sys.modules["hermes_plugins"] = ns
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"hermes_plugins.security_guidance",
|
||||
plugin_dir / "__init__.py",
|
||||
submodule_search_locations=[str(plugin_dir)],
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
mod.__package__ = "hermes_plugins.security_guidance"
|
||||
mod.__path__ = [str(plugin_dir)]
|
||||
sys.modules["hermes_plugins.security_guidance"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# patterns.py data integrity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPatternsData:
|
||||
def test_has_at_least_one_rule(self):
|
||||
p = _load_patterns()
|
||||
assert len(p.SECURITY_PATTERNS) >= 1
|
||||
|
||||
def test_every_rule_has_required_fields(self):
|
||||
p = _load_patterns()
|
||||
for rule in p.SECURITY_PATTERNS:
|
||||
assert "ruleName" in rule
|
||||
assert "reminder" in rule and rule["reminder"]
|
||||
# At least one of substrings/regex/path_check must be present —
|
||||
# otherwise the rule could never fire.
|
||||
assert any(k in rule for k in ("substrings", "regex", "path_check")), rule
|
||||
|
||||
def test_rule_names_are_unique(self):
|
||||
p = _load_patterns()
|
||||
names = [r["ruleName"] for r in p.SECURITY_PATTERNS]
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
def test_rule_id_enum_in_sync(self):
|
||||
# The upstream patterns.py asserts this at import time. If the
|
||||
# set diverges, the import itself raises and this test fails.
|
||||
p = _load_patterns()
|
||||
rule_names = {r["ruleName"] for r in p.SECURITY_PATTERNS}
|
||||
enum_names = set(p._RULE_NAME_TO_ID)
|
||||
assert rule_names == enum_names
|
||||
|
||||
def test_rule_names_to_mask_packs_bits(self):
|
||||
p = _load_patterns()
|
||||
# PICKLE_DESERIALIZATION = 8, EVAL_INJECTION = 4 → bits 8 and 4 set.
|
||||
mask = p.rule_names_to_mask({"pickle_deserialization", "eval_injection"})
|
||||
assert mask & (1 << p.RuleId.PICKLE_DESERIALIZATION)
|
||||
assert mask & (1 << p.RuleId.EVAL_INJECTION)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _scan_content
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestScanContent:
|
||||
def test_pickle_load_in_py_warns(self):
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content(
|
||||
"/tmp/foo.py", "import pickle\nx = pickle.load(open('p.pkl', 'rb'))\n"
|
||||
)
|
||||
names = [n for n, _ in findings]
|
||||
assert "pickle_deserialization" in names
|
||||
|
||||
def test_pickle_load_in_md_skipped_by_path_filter(self):
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content(
|
||||
"/tmp/foo.md", "import pickle\nx = pickle.load(open('p.pkl', 'rb'))\n"
|
||||
)
|
||||
assert findings == []
|
||||
|
||||
def test_method_call_eval_does_not_trip(self):
|
||||
"""model.eval() / redis.eval() / spec.eval() must not match eval_injection."""
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content("/tmp/foo.py", "model.eval()\nout = model(x)\n")
|
||||
assert "eval_injection" not in [n for n, _ in findings]
|
||||
|
||||
def test_bare_eval_in_py_warns(self):
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content("/tmp/foo.py", "result = eval(user_input)\n")
|
||||
assert "eval_injection" in [n for n, _ in findings]
|
||||
|
||||
def test_subprocess_shell_true_warns(self):
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content(
|
||||
"/tmp/foo.py", "subprocess.run('ls ' + path, shell=True)\n"
|
||||
)
|
||||
assert "python_subprocess_shell" in [n for n, _ in findings]
|
||||
|
||||
def test_dangerously_set_inner_html_warns(self):
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content(
|
||||
"/tmp/foo.tsx", "<div dangerouslySetInnerHTML={{__html: x}} />"
|
||||
)
|
||||
assert "react_dangerously_set_html" in [n for n, _ in findings]
|
||||
|
||||
def test_github_workflow_path_check_fires_on_path_alone(self):
|
||||
"""github_actions_workflow has no regex/substring — fires on path."""
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content(
|
||||
".github/workflows/test.yml", "name: CI\non: pull_request"
|
||||
)
|
||||
assert "github_actions_workflow" in [n for n, _ in findings]
|
||||
|
||||
def test_non_workflow_path_doesnt_trip_workflow_rule(self):
|
||||
mod = _load_plugin_init()
|
||||
findings = mod._scan_content("/tmp/foo.py", "name: CI")
|
||||
assert "github_actions_workflow" not in [n for n, _ in findings]
|
||||
|
||||
def test_empty_content_returns_no_findings(self):
|
||||
mod = _load_plugin_init()
|
||||
assert mod._scan_content("/tmp/foo.py", "") == []
|
||||
|
||||
def test_huge_content_skipped(self):
|
||||
mod = _load_plugin_init()
|
||||
# 1 MB of content with a dangerous pattern at the end — scanner caps
|
||||
# out at _MAX_SCAN_BYTES (256 KB), so this should return [].
|
||||
big = "x" * (1024 * 1024) + "\npickle.load(open('p.pkl', 'rb'))\n"
|
||||
assert mod._scan_content("/tmp/foo.py", big) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hooks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTransformToolResultHook:
|
||||
def test_warns_on_write_file_with_dangerous_content(self):
|
||||
mod = _load_plugin_init()
|
||||
args = {
|
||||
"path": "/tmp/foo.py",
|
||||
"content": "import pickle\nx = pickle.loads(b)\n",
|
||||
}
|
||||
result = mod._on_transform_tool_result(
|
||||
tool_name="write_file",
|
||||
args=args,
|
||||
result='{"success": true, "bytes_written": 30}',
|
||||
)
|
||||
assert isinstance(result, str)
|
||||
assert "Security guidance" in result
|
||||
assert "pickle_deserialization" in result
|
||||
# The original JSON should still be there at the start of the string.
|
||||
assert result.startswith('{"success": true')
|
||||
|
||||
def test_no_warn_on_clean_content(self):
|
||||
mod = _load_plugin_init()
|
||||
args = {"path": "/tmp/foo.py", "content": "import json\nx = json.loads(b)\n"}
|
||||
assert (
|
||||
mod._on_transform_tool_result(
|
||||
tool_name="write_file", args=args, result='{"success": true}'
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_no_warn_when_result_is_error(self):
|
||||
mod = _load_plugin_init()
|
||||
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
|
||||
# When the tool itself errored, we don't pile a security warning on
|
||||
# top — the model has bigger problems to solve.
|
||||
assert (
|
||||
mod._on_transform_tool_result(
|
||||
tool_name="write_file", args=args, result='{"error": "boom"}'
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_patch_tool_new_string_scanned(self):
|
||||
mod = _load_plugin_init()
|
||||
args = {
|
||||
"path": "/tmp/foo.py",
|
||||
"old_string": "x = 1",
|
||||
"new_string": "x = eval(user_input)",
|
||||
}
|
||||
result = mod._on_transform_tool_result(
|
||||
tool_name="patch", args=args, result='{"success": true}'
|
||||
)
|
||||
assert isinstance(result, str)
|
||||
assert "eval_injection" in result
|
||||
|
||||
def test_untargeted_tool_skipped(self):
|
||||
mod = _load_plugin_init()
|
||||
# The plugin only scans write_file/patch/skill_manage. terminal output
|
||||
# should pass through untouched.
|
||||
args = {"command": "echo pickle.load"}
|
||||
assert (
|
||||
mod._on_transform_tool_result(
|
||||
tool_name="terminal", args=args, result='{"output": "pickle.load"}'
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_disable_kill_switch(self, monkeypatch):
|
||||
mod = _load_plugin_init()
|
||||
monkeypatch.setenv("SECURITY_GUIDANCE_DISABLE", "1")
|
||||
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
|
||||
assert (
|
||||
mod._on_transform_tool_result(
|
||||
tool_name="write_file", args=args, result='{"ok": true}'
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_block_mode_makes_transform_hook_quiet(self, monkeypatch):
|
||||
"""In block mode, pre_tool_call handles the warning; the transform
|
||||
hook stays silent so we don't double-emit."""
|
||||
mod = _load_plugin_init()
|
||||
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
|
||||
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
|
||||
assert (
|
||||
mod._on_transform_tool_result(
|
||||
tool_name="write_file", args=args, result='{"ok": true}'
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
class TestPreToolCallHook:
|
||||
def test_no_block_in_warn_mode(self):
|
||||
mod = _load_plugin_init()
|
||||
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
|
||||
assert mod._on_pre_tool_call(tool_name="write_file", args=args) is None
|
||||
|
||||
def test_blocks_in_block_mode_on_dangerous_pattern(self, monkeypatch):
|
||||
mod = _load_plugin_init()
|
||||
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
|
||||
args = {"path": "/tmp/foo.py", "content": "pickle.load(f)\n"}
|
||||
out = mod._on_pre_tool_call(tool_name="write_file", args=args)
|
||||
assert isinstance(out, dict)
|
||||
assert out["action"] == "block"
|
||||
assert "pickle_deserialization" in out["message"]
|
||||
assert "SECURITY_GUIDANCE_BLOCK" in out["message"] # tells user how to disable
|
||||
|
||||
def test_no_block_in_block_mode_on_clean_content(self, monkeypatch):
|
||||
mod = _load_plugin_init()
|
||||
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
|
||||
args = {"path": "/tmp/foo.py", "content": "import json\n"}
|
||||
assert mod._on_pre_tool_call(tool_name="write_file", args=args) is None
|
||||
|
||||
def test_untargeted_tool_skipped(self, monkeypatch):
|
||||
mod = _load_plugin_init()
|
||||
monkeypatch.setenv("SECURITY_GUIDANCE_BLOCK", "1")
|
||||
args = {"command": "echo pickle.load(f)"}
|
||||
assert mod._on_pre_tool_call(tool_name="terminal", args=args) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bundled-plugin discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPluginDiscovery:
|
||||
def test_loads_via_plugin_manager(self, _isolate_env, monkeypatch):
|
||||
"""End-to-end: enable in config.yaml and verify the PluginManager
|
||||
picks it up via the standard discovery path."""
|
||||
import yaml
|
||||
|
||||
config = {"plugins": {"enabled": ["security-guidance"]}}
|
||||
(_isolate_env / "config.yaml").write_text(yaml.safe_dump(config))
|
||||
|
||||
# Wipe any cached plugin state from earlier tests in this worker.
|
||||
for k in list(sys.modules):
|
||||
if k.startswith(("hermes_plugins", "hermes_cli.plugins")):
|
||||
del sys.modules[k]
|
||||
|
||||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||||
|
||||
mgr = _ensure_plugins_discovered(force=True)
|
||||
loaded = set()
|
||||
if hasattr(mgr, "_plugins"):
|
||||
loaded = set(mgr._plugins.keys())
|
||||
assert "security-guidance" in loaded
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Tests for the Teams pipeline plugin package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest
|
||||
from gateway.config import GatewayConfig, Platform, PlatformConfig
|
||||
from plugins.teams_pipeline import register
|
||||
from plugins.teams_pipeline.pipeline import TeamsMeetingPipeline
|
||||
from plugins.teams_pipeline.store import TeamsPipelineStore
|
||||
from plugins.teams_pipeline.models import MeetingArtifact
|
||||
|
||||
|
||||
class FakeGraphClient:
|
||||
def __init__(self) -> None:
|
||||
self.downloaded = False
|
||||
|
||||
|
||||
async def _transcript_meeting_resolver(client, *, meeting_id=None, join_web_url=None, tenant_id=None):
|
||||
from plugins.teams_pipeline.models import TeamsMeetingRef
|
||||
|
||||
return TeamsMeetingRef(
|
||||
meeting_id=str(meeting_id),
|
||||
tenant_id=tenant_id,
|
||||
metadata={"subject": "Weekly Sync", "participants": [{"displayName": "Ada"}]},
|
||||
)
|
||||
|
||||
|
||||
async def _no_call_record(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def test_register_adds_cli_only():
|
||||
mgr = PluginManager()
|
||||
manifest = PluginManifest(name="teams_pipeline")
|
||||
ctx = PluginContext(manifest, mgr)
|
||||
|
||||
register(ctx)
|
||||
|
||||
assert "teams-pipeline" in mgr._cli_commands
|
||||
entry = mgr._cli_commands["teams-pipeline"]
|
||||
assert entry["plugin"] == "teams_pipeline"
|
||||
assert callable(entry["setup_fn"])
|
||||
assert callable(entry["handler_fn"])
|
||||
|
||||
|
||||
def test_runtime_config_uses_existing_teams_platform_settings():
|
||||
from plugins.teams_pipeline.runtime import build_pipeline_runtime_config
|
||||
|
||||
gateway_config = GatewayConfig(
|
||||
platforms={
|
||||
Platform("teams"): PlatformConfig(
|
||||
enabled=True,
|
||||
extra={
|
||||
"delivery_mode": "graph",
|
||||
"team_id": "team-1",
|
||||
"channel_id": "channel-1",
|
||||
"meeting_pipeline": {
|
||||
"transcript_min_chars": 120,
|
||||
"notion": {"enabled": True, "database_id": "db-1"},
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
runtime_config = build_pipeline_runtime_config(gateway_config)
|
||||
|
||||
assert runtime_config["transcript_min_chars"] == 120
|
||||
assert runtime_config["notion"]["database_id"] == "db-1"
|
||||
assert runtime_config["teams_delivery"] == {
|
||||
"enabled": True,
|
||||
"mode": "graph",
|
||||
"team_id": "team-1",
|
||||
"channel_id": "channel-1",
|
||||
}
|
||||
|
||||
|
||||
def test_build_pipeline_runtime_reuses_existing_teams_adapter_surface(monkeypatch, tmp_path):
|
||||
from plugins.teams_pipeline import runtime as runtime_module
|
||||
|
||||
class FakeWriter:
|
||||
def __init__(self, platform_config=None, **kwargs) -> None:
|
||||
self.platform_config = platform_config
|
||||
|
||||
monkeypatch.setattr(runtime_module, "build_graph_client", lambda: object())
|
||||
monkeypatch.setattr(runtime_module, "resolve_teams_pipeline_store_path", lambda: tmp_path / "teams-store.json")
|
||||
monkeypatch.setattr("plugins.platforms.teams.adapter.TeamsSummaryWriter", FakeWriter)
|
||||
|
||||
gateway = SimpleNamespace(
|
||||
config=GatewayConfig(
|
||||
platforms={
|
||||
Platform("teams"): PlatformConfig(
|
||||
enabled=True,
|
||||
extra={
|
||||
"delivery_mode": "incoming_webhook",
|
||||
"incoming_webhook_url": "https://example.com/hook",
|
||||
},
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
runtime = runtime_module.build_pipeline_runtime(gateway)
|
||||
|
||||
assert isinstance(runtime.teams_sender, FakeWriter)
|
||||
assert runtime.teams_sender.platform_config is gateway.config.platforms[Platform("teams")]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_bind_gateway_runtime_attaches_scheduler(monkeypatch, tmp_path):
|
||||
from plugins.teams_pipeline import runtime as runtime_module
|
||||
|
||||
class FakeAdapter:
|
||||
def __init__(self) -> None:
|
||||
self.scheduler = None
|
||||
|
||||
def set_notification_scheduler(self, scheduler) -> None:
|
||||
self.scheduler = scheduler
|
||||
|
||||
class FakePipeline:
|
||||
def __init__(self) -> None:
|
||||
self.notifications = []
|
||||
|
||||
async def run_notification(self, notification):
|
||||
self.notifications.append(notification)
|
||||
|
||||
adapter = FakeAdapter()
|
||||
pipeline = FakePipeline()
|
||||
gateway = SimpleNamespace(
|
||||
adapters={Platform.MSGRAPH_WEBHOOK: adapter},
|
||||
config=GatewayConfig(platforms={}),
|
||||
_teams_pipeline_runtime=None,
|
||||
_teams_pipeline_runtime_error=None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(runtime_module, "build_pipeline_runtime", lambda gateway_runner: pipeline)
|
||||
|
||||
bound = runtime_module.bind_gateway_runtime(gateway)
|
||||
|
||||
assert bound is True
|
||||
assert gateway._teams_pipeline_runtime is pipeline
|
||||
assert callable(adapter.scheduler)
|
||||
|
||||
notification = {"id": "notif-1"}
|
||||
await adapter.scheduler(notification, object())
|
||||
assert pipeline.notifications == [notification]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_bind_gateway_runtime_drops_notifications_when_unavailable(monkeypatch):
|
||||
from plugins.teams_pipeline import runtime as runtime_module
|
||||
from tools.microsoft_graph_auth import MicrosoftGraphConfigError
|
||||
|
||||
class FakeAdapter:
|
||||
def __init__(self) -> None:
|
||||
self.scheduler = None
|
||||
|
||||
def set_notification_scheduler(self, scheduler) -> None:
|
||||
self.scheduler = scheduler
|
||||
|
||||
adapter = FakeAdapter()
|
||||
gateway = SimpleNamespace(
|
||||
adapters={Platform.MSGRAPH_WEBHOOK: adapter},
|
||||
config=GatewayConfig(platforms={}),
|
||||
_teams_pipeline_runtime=None,
|
||||
_teams_pipeline_runtime_error=None,
|
||||
)
|
||||
|
||||
def _raise(_gateway_runner):
|
||||
raise MicrosoftGraphConfigError("missing graph env")
|
||||
|
||||
monkeypatch.setattr(runtime_module, "build_pipeline_runtime", _raise)
|
||||
|
||||
bound = runtime_module.bind_gateway_runtime(gateway)
|
||||
|
||||
assert bound is False
|
||||
assert "missing graph env" in gateway._teams_pipeline_runtime_error
|
||||
assert callable(adapter.scheduler)
|
||||
await adapter.scheduler({"id": "notif-2"}, object())
|
||||
|
||||
|
||||
def test_store_persists_subscription_event_and_job_state(tmp_path):
|
||||
store_path = tmp_path / "teams-store.json"
|
||||
store = TeamsPipelineStore(store_path)
|
||||
store.upsert_subscription(
|
||||
"sub-1",
|
||||
{"client_state": "abc", "resource": "communications/onlineMeetings"},
|
||||
)
|
||||
store.record_event_timestamp("evt-1", "2026-05-03T19:30:00Z")
|
||||
store.upsert_job("job-1", {"status": "received", "event_id": "evt-1"})
|
||||
store.upsert_sink_record("notion:meeting-1", {"page_id": "page-1"})
|
||||
|
||||
reloaded = TeamsPipelineStore(store_path)
|
||||
subscription = reloaded.get_subscription("sub-1")
|
||||
job = reloaded.get_job("job-1")
|
||||
sink = reloaded.get_sink_record("notion:meeting-1")
|
||||
|
||||
assert subscription is not None
|
||||
assert subscription["subscription_id"] == "sub-1"
|
||||
assert subscription["client_state"] == "abc"
|
||||
assert reloaded.get_event_timestamp("evt-1") == "2026-05-03T19:30:00Z"
|
||||
assert job is not None
|
||||
assert job["status"] == "received"
|
||||
assert sink is not None
|
||||
assert sink["page_id"] == "page-1"
|
||||
|
||||
|
||||
def test_store_notification_receipts_are_idempotent(tmp_path):
|
||||
store = TeamsPipelineStore(tmp_path / "teams-store.json")
|
||||
notification = {
|
||||
"subscriptionId": "sub-1",
|
||||
"resource": "communications/onlineMeetings/meeting-1",
|
||||
"changeType": "updated",
|
||||
}
|
||||
receipt_key = TeamsPipelineStore.build_notification_receipt_key(notification)
|
||||
|
||||
assert store.record_notification_receipt(receipt_key, notification) is True
|
||||
assert store.record_notification_receipt(receipt_key, notification) is False
|
||||
assert store.has_notification_receipt(receipt_key) is True
|
||||
|
||||
reloaded = TeamsPipelineStore(tmp_path / "teams-store.json")
|
||||
assert reloaded.has_notification_receipt(receipt_key) is True
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
class TestTeamsMeetingPipeline:
|
||||
async def test_transcript_first_path_persists_state_and_skips_recording(self, tmp_path, monkeypatch):
|
||||
from plugins.teams_pipeline import pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
|
||||
|
||||
async def _fetch_transcript(client, meeting_ref):
|
||||
return (
|
||||
MeetingArtifact(artifact_type="transcript", artifact_id="tx-1", display_name="meeting.vtt"),
|
||||
"Action: Send draft by Friday.\nDecision: Ship the transcript-first path.\nDetailed transcript content.",
|
||||
)
|
||||
|
||||
async def _call_record(client, meeting_ref, *, call_record_id=None, allow_permission_errors=True):
|
||||
return MeetingArtifact(
|
||||
artifact_type="call_record",
|
||||
artifact_id="call-1",
|
||||
metadata={"metrics": {"participant_count": 4}},
|
||||
)
|
||||
|
||||
async def _summarize(**kwargs):
|
||||
return pipeline_module.TeamsMeetingSummaryPayload(
|
||||
meeting_ref=kwargs["resolved_meeting"],
|
||||
title="Weekly Sync",
|
||||
transcript_text=kwargs["transcript_text"],
|
||||
summary="Short summary",
|
||||
key_decisions=["Ship the transcript-first path."],
|
||||
action_items=["Send draft by Friday."],
|
||||
risks=["Timeline risk."],
|
||||
confidence="high",
|
||||
confidence_notes="Transcript available.",
|
||||
source_artifacts=kwargs["artifacts"],
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _fetch_transcript)
|
||||
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _call_record)
|
||||
|
||||
store = TeamsPipelineStore(tmp_path / "teams-store.json")
|
||||
pipeline = TeamsMeetingPipeline(
|
||||
graph_client=FakeGraphClient(),
|
||||
store=store,
|
||||
config={"transcript_min_chars": 20},
|
||||
summarize_fn=_summarize,
|
||||
)
|
||||
|
||||
job = await pipeline.run_notification(
|
||||
{
|
||||
"id": "notif-1",
|
||||
"changeType": "updated",
|
||||
"resource": "communications/onlineMeetings/meeting-123",
|
||||
"resourceData": {"id": "meeting-123"},
|
||||
}
|
||||
)
|
||||
|
||||
assert job.status == "completed"
|
||||
assert job.selected_artifact_strategy == "transcript_first"
|
||||
assert job.summary_payload is not None
|
||||
assert job.summary_payload.summary == "Short summary"
|
||||
stored = store.get_job(job.job_id)
|
||||
assert stored is not None
|
||||
assert stored["status"] == "completed"
|
||||
|
||||
async def test_recording_fallback_uses_stt_and_updates_sink_records(self, tmp_path, monkeypatch):
|
||||
from plugins.teams_pipeline import pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
|
||||
|
||||
async def _no_transcript(client, meeting_ref):
|
||||
return None, None
|
||||
|
||||
async def _recordings(client, meeting_ref):
|
||||
return [
|
||||
MeetingArtifact(
|
||||
artifact_type="recording",
|
||||
artifact_id="rec-1",
|
||||
display_name="recording.mp4",
|
||||
download_url="https://files.example/recording.mp4",
|
||||
)
|
||||
]
|
||||
|
||||
async def _download(client, meeting_ref, recording, destination):
|
||||
target = Path(destination)
|
||||
target.write_bytes(b"video-bytes")
|
||||
return {"path": str(target), "size_bytes": 11, "content_type": "video/mp4"}
|
||||
|
||||
async def _prepare_audio(self, recording_path):
|
||||
audio_path = recording_path.with_suffix(".wav")
|
||||
audio_path.write_bytes(b"audio-bytes")
|
||||
return audio_path
|
||||
|
||||
def _transcribe(file_path, model):
|
||||
return {"success": True, "transcript": "Action: Follow up with Legal.\nRisk: Budget approval pending.", "provider": "local"}
|
||||
|
||||
async def _summarize(**kwargs):
|
||||
return pipeline_module.TeamsMeetingSummaryPayload(
|
||||
meeting_ref=kwargs["resolved_meeting"],
|
||||
title="Weekly Sync",
|
||||
transcript_text=kwargs["transcript_text"],
|
||||
summary="Fallback summary",
|
||||
key_decisions=[],
|
||||
action_items=["Follow up with Legal."],
|
||||
risks=["Budget approval pending."],
|
||||
confidence="medium",
|
||||
confidence_notes="Generated from STT fallback.",
|
||||
source_artifacts=kwargs["artifacts"],
|
||||
)
|
||||
|
||||
class FakeNotionWriter:
|
||||
async def write_summary(self, payload, config, existing_record=None):
|
||||
return {"page_id": existing_record.get("page_id") if existing_record else "page-1", "url": "https://notion.so/page-1"}
|
||||
|
||||
async def _teams_sender(payload, config, existing_record=None):
|
||||
return {"message_id": existing_record.get("message_id") if existing_record else "msg-1"}
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _no_transcript)
|
||||
monkeypatch.setattr(pipeline_module, "list_recording_artifacts", _recordings)
|
||||
monkeypatch.setattr(pipeline_module, "download_recording_artifact", _download)
|
||||
monkeypatch.setattr(pipeline_module.TeamsMeetingPipeline, "_prepare_audio_path", _prepare_audio)
|
||||
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _no_call_record)
|
||||
|
||||
store = TeamsPipelineStore(tmp_path / "teams-store.json")
|
||||
pipeline = TeamsMeetingPipeline(
|
||||
graph_client=FakeGraphClient(),
|
||||
store=store,
|
||||
config={
|
||||
"notion": {"enabled": True, "database_id": "db-1"},
|
||||
"teams_delivery": {"enabled": True, "channel_id": "channel-1"},
|
||||
},
|
||||
transcribe_fn=_transcribe,
|
||||
summarize_fn=_summarize,
|
||||
notion_writer=FakeNotionWriter(),
|
||||
teams_sender=_teams_sender,
|
||||
)
|
||||
|
||||
job = await pipeline.run_notification(
|
||||
{
|
||||
"id": "notif-2",
|
||||
"changeType": "updated",
|
||||
"resource": "communications/onlineMeetings/meeting-456",
|
||||
"resourceData": {"id": "meeting-456"},
|
||||
}
|
||||
)
|
||||
|
||||
assert job.status == "completed"
|
||||
assert job.selected_artifact_strategy == "recording_stt_fallback"
|
||||
assert job.summary_payload is not None
|
||||
assert job.summary_payload.summary == "Fallback summary"
|
||||
notion_record = store.get_sink_record("notion:meeting-456")
|
||||
teams_record = store.get_sink_record("teams:meeting-456")
|
||||
assert notion_record is not None
|
||||
assert notion_record["page_id"] == "page-1"
|
||||
assert teams_record is not None
|
||||
assert teams_record["message_id"] == "msg-1"
|
||||
|
||||
async def test_missing_transcript_and_recording_schedules_retry(self, tmp_path, monkeypatch):
|
||||
from plugins.teams_pipeline import pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
|
||||
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", lambda *a, **kw: asyncio.sleep(0, result=(None, None)))
|
||||
monkeypatch.setattr(pipeline_module, "list_recording_artifacts", lambda *a, **kw: asyncio.sleep(0, result=[]))
|
||||
|
||||
store = TeamsPipelineStore(tmp_path / "teams-store.json")
|
||||
pipeline = TeamsMeetingPipeline(
|
||||
graph_client=FakeGraphClient(),
|
||||
store=store,
|
||||
config={},
|
||||
summarize_fn=lambda **kwargs: asyncio.sleep(0, result=None),
|
||||
)
|
||||
|
||||
job = await pipeline.run_notification(
|
||||
{
|
||||
"id": "notif-3",
|
||||
"changeType": "updated",
|
||||
"resource": "communications/onlineMeetings/meeting-789",
|
||||
"resourceData": {"id": "meeting-789"},
|
||||
}
|
||||
)
|
||||
|
||||
assert job.status == "retry_scheduled"
|
||||
assert job.error_info["retryable"] is True
|
||||
assert "Recording unavailable" in job.error_info["message"]
|
||||
|
||||
async def test_duplicate_notification_reuses_completed_job(self, tmp_path, monkeypatch):
|
||||
from plugins.teams_pipeline import pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "resolve_meeting_reference", _transcript_meeting_resolver)
|
||||
|
||||
async def _fetch_transcript(client, meeting_ref):
|
||||
return (
|
||||
MeetingArtifact(artifact_type="transcript", artifact_id="tx-dup", display_name="meeting.vtt"),
|
||||
"Decision: Keep duplicate notifications idempotent.\nAction: Verify the cached job is reused.",
|
||||
)
|
||||
|
||||
summarize_calls = 0
|
||||
|
||||
async def _summarize(**kwargs):
|
||||
nonlocal summarize_calls
|
||||
summarize_calls += 1
|
||||
return pipeline_module.TeamsMeetingSummaryPayload(
|
||||
meeting_ref=kwargs["resolved_meeting"],
|
||||
title="Weekly Sync",
|
||||
transcript_text=kwargs["transcript_text"],
|
||||
summary="Duplicate-safe summary",
|
||||
key_decisions=["Keep duplicate notifications idempotent."],
|
||||
action_items=["Verify the cached job is reused."],
|
||||
confidence="high",
|
||||
confidence_notes="Transcript available.",
|
||||
source_artifacts=kwargs["artifacts"],
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "fetch_preferred_transcript_text", _fetch_transcript)
|
||||
monkeypatch.setattr(pipeline_module, "enrich_meeting_with_call_record", _no_call_record)
|
||||
|
||||
store = TeamsPipelineStore(tmp_path / "teams-store.json")
|
||||
pipeline = TeamsMeetingPipeline(
|
||||
graph_client=FakeGraphClient(),
|
||||
store=store,
|
||||
config={"transcript_min_chars": 20},
|
||||
summarize_fn=_summarize,
|
||||
)
|
||||
notification = {
|
||||
"id": "notif-dup",
|
||||
"changeType": "updated",
|
||||
"resource": "communications/onlineMeetings/meeting-dup",
|
||||
"resourceData": {"id": "meeting-dup"},
|
||||
}
|
||||
|
||||
first_job = await pipeline.run_notification(notification)
|
||||
second_job = await pipeline.run_notification(notification)
|
||||
|
||||
assert first_job.status == "completed"
|
||||
assert second_job.status == "completed"
|
||||
assert second_job.job_id == first_job.job_id
|
||||
assert summarize_calls == 1
|
||||
assert len(store.list_jobs()) == 1
|
||||
receipt_key = TeamsPipelineStore.build_notification_receipt_key(notification)
|
||||
assert store.has_notification_receipt(receipt_key) is True
|
||||
@@ -0,0 +1,431 @@
|
||||
"""Behavior-parity check for the STT plugin hook + command-provider registry.
|
||||
|
||||
Spawns one subprocess per (version, scenario) cell — pinned to either
|
||||
``origin/main`` (no plugin hook, no STT command-provider registry; only
|
||||
the legacy ``HERMES_LOCAL_STT_COMMAND`` escape hatch exists) or this PR's
|
||||
worktree (both new surfaces present).
|
||||
|
||||
Each subprocess clears all STT-related env vars + writes a
|
||||
``config.yaml``, then asks the dispatcher how it would route a
|
||||
``transcribe_audio`` call. The emitted shape tuple is::
|
||||
|
||||
{dispatch_kind, provider_name, success}
|
||||
|
||||
Where ``dispatch_kind`` ∈
|
||||
``{"builtin_local", "builtin_groq", "builtin_openai", ...,
|
||||
"plugin", "plugin_unavailable", "command_provider",
|
||||
"no_provider_error", "stt_disabled"}``.
|
||||
|
||||
Acceptable diffs:
|
||||
- ``no_provider_error → plugin`` for the ``plugin-installed`` scenario.
|
||||
- ``no_provider_error → plugin_unavailable`` for the
|
||||
``plugin-installed-unavailable`` scenario (PR returns the cleaner
|
||||
unavailability envelope instead of the generic auto-detect error).
|
||||
- ``no_provider_error → command_provider`` for the
|
||||
``command-provider-installed`` scenario (registry shipped with this PR).
|
||||
- ``no_provider_error → command_provider`` for
|
||||
``command-vs-plugin-same-name`` (command wins precedence, same as TTS).
|
||||
|
||||
Run from the PR worktree::
|
||||
|
||||
python tests/plugins/transcription/check_parity_vs_main.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _resolve_main_dir() -> Path:
|
||||
candidate = REPO_ROOT.parent.parent
|
||||
if (candidate / "tools" / "transcription_tools.py").exists() and candidate != REPO_ROOT:
|
||||
return candidate
|
||||
sibling = REPO_ROOT.parent / "hermes-agent-main"
|
||||
if (sibling / "tools" / "transcription_tools.py").exists():
|
||||
return sibling
|
||||
return REPO_ROOT
|
||||
|
||||
|
||||
MAIN_DIR = _resolve_main_dir()
|
||||
PR_DIR = REPO_ROOT
|
||||
assert (PR_DIR / "tools" / "transcription_tools.py").exists(), (
|
||||
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
|
||||
)
|
||||
|
||||
|
||||
SUBPROCESS_SCRIPT = r"""
|
||||
import json, os, sys, tempfile
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
|
||||
# Isolated HERMES_HOME so the config write is hermetic.
|
||||
home = tempfile.mkdtemp()
|
||||
os.environ["HERMES_HOME"] = home
|
||||
|
||||
# Clear STT-related env so dispatch decisions are config-driven.
|
||||
for k in (
|
||||
"GROQ_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY",
|
||||
"MISTRAL_API_KEY", "XAI_API_KEY",
|
||||
"HERMES_LOCAL_STT_COMMAND",
|
||||
):
|
||||
os.environ.pop(k, None)
|
||||
|
||||
scenario_env = json.loads(sys.argv[2])
|
||||
os.environ.update(scenario_env)
|
||||
|
||||
config_yaml = sys.argv[3]
|
||||
plugin_register = sys.argv[4] # "yes" to register a fake plugin
|
||||
|
||||
config_path = os.path.join(home, "config.yaml")
|
||||
with open(config_path, "w") as f:
|
||||
f.write(config_yaml)
|
||||
|
||||
# Fresh import — must not have anything cached from prior runs.
|
||||
for name in list(sys.modules):
|
||||
if (name.startswith("tools.")
|
||||
or name.startswith("agent.")
|
||||
or name.startswith("plugins.")
|
||||
or name.startswith("hermes_cli.")):
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
# Try importing transcription_registry — only exists on PR side.
|
||||
have_plugin_hook = False
|
||||
try:
|
||||
from agent import transcription_registry
|
||||
from agent.transcription_provider import TranscriptionProvider
|
||||
have_plugin_hook = True
|
||||
|
||||
if plugin_register == "yes":
|
||||
class _FakeProvider(TranscriptionProvider):
|
||||
@property
|
||||
def name(self): return "openrouter"
|
||||
def transcribe(self, file_path, **kw):
|
||||
return {"success": True, "transcript": "PLUGIN: openrouter transcript", "provider": "openrouter"}
|
||||
|
||||
transcription_registry._reset_for_tests()
|
||||
transcription_registry.register_provider(_FakeProvider())
|
||||
elif plugin_register == "unavailable":
|
||||
class _UnavailablePlugin(TranscriptionProvider):
|
||||
@property
|
||||
def name(self): return "openrouter"
|
||||
def is_available(self): return False
|
||||
def transcribe(self, file_path, **kw):
|
||||
return {"success": True, "transcript": "should not run"}
|
||||
|
||||
transcription_registry._reset_for_tests()
|
||||
transcription_registry.register_provider(_UnavailablePlugin())
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import tools.transcription_tools as tt
|
||||
|
||||
# Use a real (but empty) audio file so _validate_audio_file passes.
|
||||
audio_path = os.path.join(home, "audio.ogg")
|
||||
with open(audio_path, "wb") as f:
|
||||
# Minimal-ish OGG-shaped bytes so the size check passes.
|
||||
f.write(b"OggS" + b"\x00" * 1024)
|
||||
|
||||
# Patch _transcribe_* so the test doesn't actually try cloud APIs.
|
||||
# We're testing dispatch, not the underlying transcription.
|
||||
def _stub(file_path, model_name=None):
|
||||
return {"success": True, "transcript": "stub from " + sys._getframe().f_code.co_name.replace("_stub_", ""),
|
||||
"provider": sys._getframe().f_code.co_name.replace("_stub_", "")}
|
||||
|
||||
# Stub each built-in to a marker so we can identify the branch.
|
||||
class _Stub:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
def __call__(self, file_path, model_name=None):
|
||||
return {"success": True, "transcript": "stub", "provider": self.name}
|
||||
|
||||
tt._transcribe_local = _Stub("local")
|
||||
tt._transcribe_local_command = _Stub("local_command")
|
||||
tt._transcribe_groq = _Stub("groq")
|
||||
tt._transcribe_openai = _Stub("openai")
|
||||
tt._transcribe_mistral = _Stub("mistral")
|
||||
tt._transcribe_xai = _Stub("xai")
|
||||
|
||||
# Force _get_provider to honor the explicit config since we don't have
|
||||
# real creds. The provider-resolution gates check _HAS_OPENAI /
|
||||
# _HAS_FASTER_WHISPER which we can't easily set, so we just patch
|
||||
# _get_provider to return whatever the config says.
|
||||
stt_cfg = tt._load_stt_config()
|
||||
explicit = stt_cfg.get("provider")
|
||||
if explicit:
|
||||
# Bypass the gating for test purposes — _get_provider would
|
||||
# otherwise return "none" when the dependency isn't installed.
|
||||
original_get = tt._get_provider
|
||||
def _patched(cfg):
|
||||
if not tt.is_stt_enabled(cfg):
|
||||
return "none"
|
||||
return cfg.get("provider", "none")
|
||||
tt._get_provider = _patched
|
||||
|
||||
try:
|
||||
result = tt.transcribe_audio(audio_path)
|
||||
except Exception as exc:
|
||||
shape = {"dispatch_kind": "exception", "provider_name": None, "success": False,
|
||||
"error_text": repr(exc)}
|
||||
print(json.dumps(shape))
|
||||
sys.exit(0)
|
||||
|
||||
dispatch_kind = "unknown"
|
||||
provider_name = result.get("provider") if isinstance(result, dict) else None
|
||||
success = result.get("success", False) if isinstance(result, dict) else False
|
||||
error_text = result.get("error", "") if isinstance(result, dict) else ""
|
||||
|
||||
if not success and "STT is disabled" in error_text:
|
||||
dispatch_kind = "stt_disabled"
|
||||
elif not success and "is not available" in error_text:
|
||||
dispatch_kind = "plugin_unavailable"
|
||||
elif not success and "No STT provider" in error_text:
|
||||
dispatch_kind = "no_provider_error"
|
||||
elif provider_name in ("local", "local_command", "groq", "openai", "mistral", "xai"):
|
||||
dispatch_kind = "builtin_" + provider_name
|
||||
elif success and isinstance(result, dict) and result.get("transcript", "").startswith("CMD:"):
|
||||
# Command-provider scenarios below emit transcripts prefixed with "CMD:"
|
||||
# so the harness can distinguish command-provider dispatch from a
|
||||
# plugin dispatch even when they share a provider name.
|
||||
dispatch_kind = "command_provider"
|
||||
elif success and isinstance(result, dict) and result.get("transcript", "").startswith("PLUGIN:"):
|
||||
dispatch_kind = "plugin"
|
||||
elif success and provider_name and provider_name not in ("local", "local_command", "groq", "openai", "mistral", "xai"):
|
||||
dispatch_kind = "plugin"
|
||||
else:
|
||||
dispatch_kind = "other"
|
||||
|
||||
shape = {
|
||||
"dispatch_kind": dispatch_kind,
|
||||
"provider_name": provider_name,
|
||||
"success": success,
|
||||
}
|
||||
print(json.dumps(shape))
|
||||
"""
|
||||
|
||||
|
||||
def _cmd_yaml(provider_name: str, transcript: str) -> str:
|
||||
"""Build a YAML snippet for an stt.providers.<name>: type: command entry.
|
||||
|
||||
Produces a shell command that writes ``transcript`` to {output_path}.
|
||||
Backslashes in the venv python path are doubled for YAML, and the
|
||||
inner double quotes around the python -c payload are YAML-escaped.
|
||||
Keeps the test scenarios readable.
|
||||
"""
|
||||
interp = sys.executable.replace("\\", "\\\\")
|
||||
# Inside the YAML double-quoted string, we use single quotes around
|
||||
# the python -c body so we don't have to YAML-escape inner double
|
||||
# quotes. Single quotes inside the body are not needed; the body uses
|
||||
# double quotes for module references and string literals.
|
||||
payload = (
|
||||
f"import sys; open(sys.argv[1], 'w').write('{transcript}')"
|
||||
)
|
||||
command = f'{interp} -c "{payload}" {{output_path}}'
|
||||
# YAML-escape: double-quote the whole thing, escape inner " and \.
|
||||
yaml_escaped = command.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return (
|
||||
"stt:\n"
|
||||
f" provider: {provider_name}\n"
|
||||
" providers:\n"
|
||||
f" {provider_name}:\n"
|
||||
" type: command\n"
|
||||
f' command: "{yaml_escaped}"\n'
|
||||
)
|
||||
|
||||
|
||||
SCENARIOS: list[tuple[str, str, dict[str, str], str]] = [
|
||||
# (label, config.yaml body, scenario_env, plugin_register)
|
||||
("stt-disabled", "stt:\n enabled: false\n", {}, "no"),
|
||||
("explicit-groq", "stt:\n provider: groq\n", {}, "no"),
|
||||
("explicit-openai", "stt:\n provider: openai\n", {}, "no"),
|
||||
("explicit-local", "stt:\n provider: local\n", {}, "no"),
|
||||
("explicit-xai", "stt:\n provider: xai\n", {}, "no"),
|
||||
# Mistral is quarantined → _get_provider returns "none" today, hence no_provider_error.
|
||||
("explicit-mistral-quarantine", "stt:\n provider: mistral\n", {}, "no"),
|
||||
# Unknown name + no plugin → both: no_provider_error
|
||||
("unknown-no-plugin", "stt:\n provider: openrouter\n", {}, "no"),
|
||||
# Unknown name + plugin installed → main: no_provider_error, PR: plugin
|
||||
("plugin-installed", "stt:\n provider: openrouter\n", {}, "yes"),
|
||||
# Unknown name + plugin reports unavailable → main: no_provider_error,
|
||||
# PR: plugin_unavailable (cleaner envelope, names the plugin)
|
||||
("plugin-installed-unavailable", "stt:\n provider: openrouter\n", {}, "unavailable"),
|
||||
# Built-in name + plugin tries to shadow → both: built-in
|
||||
("explicit-openai-with-plugin-registered", "stt:\n provider: openai\n", {}, "yes"),
|
||||
# NEW (this PR): stt.providers.<name>: type: command registry.
|
||||
# Provider name "fake-cli" + transcript prefixed "CMD:" so dispatch_kind
|
||||
# detection routes it to "command_provider". On main (no registry),
|
||||
# this falls through to no_provider_error.
|
||||
(
|
||||
"command-provider-installed",
|
||||
_cmd_yaml("fake-cli", "CMD: fake-cli transcript"),
|
||||
{},
|
||||
"no",
|
||||
),
|
||||
# NEW (this PR): same name registered as BOTH a command provider and
|
||||
# a plugin under "openrouter". Command must win (config more local
|
||||
# than plugin install). The plugin emits "PLUGIN:..." — assertion is
|
||||
# that the transcript is "CMD:...", proving command-wins precedence.
|
||||
(
|
||||
"command-vs-plugin-same-name",
|
||||
_cmd_yaml("openrouter", "CMD: openrouter via command wins"),
|
||||
{},
|
||||
"yes", # also register a plugin under "openrouter" — must NOT fire
|
||||
),
|
||||
# NEW (this PR): built-in name with a command provider declared under
|
||||
# it → built-in still wins (built-in elif chain has precedence).
|
||||
# The command would write "CMD: HIJACK" if it fired — assertion is
|
||||
# that built-in OpenAI dispatch fires instead.
|
||||
(
|
||||
"explicit-openai-with-command-shadow",
|
||||
_cmd_yaml("openai", "CMD: HIJACK"),
|
||||
{},
|
||||
"no",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# Subprocesses reset the registry between runs via ``_reset_for_tests`` so
|
||||
# registrations from earlier scenarios don't leak. The command-provider
|
||||
# scenarios also work on origin/main — the subprocess just executes the
|
||||
# native dispatch path, which falls through to "no_provider_error" because
|
||||
# main has no registry for stt.providers.<name>.
|
||||
|
||||
|
||||
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict, plugin_register: str) -> dict:
|
||||
venv_python = repo_path / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = MAIN_DIR / "venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = Path("python3")
|
||||
|
||||
out = subprocess.run(
|
||||
[
|
||||
str(venv_python),
|
||||
"-c",
|
||||
SUBPROCESS_SCRIPT,
|
||||
str(repo_path),
|
||||
json.dumps(env),
|
||||
config_yaml,
|
||||
plugin_register,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return {
|
||||
"error": "subprocess failed",
|
||||
"stdout": out.stdout[-500:],
|
||||
"stderr": out.stderr[-500:],
|
||||
}
|
||||
try:
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
except Exception as exc:
|
||||
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
|
||||
|
||||
|
||||
def _reduce(shape: dict) -> dict:
|
||||
return {
|
||||
"dispatch_kind": shape.get("dispatch_kind"),
|
||||
"success": shape.get("success"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print(f"main: {MAIN_DIR}")
|
||||
print(f"pr: {PR_DIR}")
|
||||
print()
|
||||
|
||||
if MAIN_DIR == PR_DIR:
|
||||
print(
|
||||
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
|
||||
" Set up a sibling 'hermes-agent-main' checkout pinned to "
|
||||
"origin/main to get real parity coverage."
|
||||
)
|
||||
print()
|
||||
|
||||
failures: list[str] = []
|
||||
errors: list[str] = []
|
||||
intentional_diffs: list[tuple[str, dict, dict]] = []
|
||||
for label, config_yaml, env, plugin_register in SCENARIOS:
|
||||
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env, plugin_register)
|
||||
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env, plugin_register)
|
||||
|
||||
if "error" in main_shape or "error" in pr_shape:
|
||||
print(f" [ERR ] {label}: subprocess failed")
|
||||
print(f" main: {main_shape}")
|
||||
print(f" pr: {pr_shape}")
|
||||
errors.append(label)
|
||||
continue
|
||||
|
||||
main_reduced = _reduce(main_shape)
|
||||
pr_reduced = _reduce(pr_shape)
|
||||
|
||||
if main_reduced == pr_reduced:
|
||||
print(f" [OK] {label}: {main_reduced}")
|
||||
continue
|
||||
|
||||
# On main, "plugin-installed" returns no_provider_error (no
|
||||
# plugin hook); on PR, plugin dispatches. Same shape for
|
||||
# "plugin-installed-unavailable" but PR returns the cleaner
|
||||
# plugin_unavailable envelope. The new command-provider scenarios
|
||||
# also intentionally diff against main (which has no stt.providers
|
||||
# registry yet).
|
||||
no_provider_to_plugin = (
|
||||
main_reduced.get("dispatch_kind") == "no_provider_error"
|
||||
and pr_reduced.get("dispatch_kind") == "plugin"
|
||||
and label == "plugin-installed"
|
||||
)
|
||||
no_provider_to_unavailable = (
|
||||
main_reduced.get("dispatch_kind") == "no_provider_error"
|
||||
and pr_reduced.get("dispatch_kind") == "plugin_unavailable"
|
||||
and label == "plugin-installed-unavailable"
|
||||
)
|
||||
no_provider_to_command = (
|
||||
main_reduced.get("dispatch_kind") == "no_provider_error"
|
||||
and pr_reduced.get("dispatch_kind") == "command_provider"
|
||||
and label in {"command-provider-installed", "command-vs-plugin-same-name"}
|
||||
)
|
||||
if no_provider_to_plugin:
|
||||
print(f" [DIFF] {label}: no_provider_error → plugin — expected")
|
||||
intentional_diffs.append((label, main_reduced, pr_reduced))
|
||||
elif no_provider_to_unavailable:
|
||||
print(f" [DIFF] {label}: no_provider_error → plugin_unavailable — expected")
|
||||
intentional_diffs.append((label, main_reduced, pr_reduced))
|
||||
elif no_provider_to_command:
|
||||
print(f" [DIFF] {label}: no_provider_error → command_provider — expected")
|
||||
intentional_diffs.append((label, main_reduced, pr_reduced))
|
||||
else:
|
||||
print(f" [FAIL] {label}")
|
||||
print(f" main: {main_reduced}")
|
||||
print(f" pr: {pr_reduced}")
|
||||
failures.append(label)
|
||||
|
||||
print()
|
||||
if errors:
|
||||
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
if failures:
|
||||
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
|
||||
for f in failures:
|
||||
print(f" - {f}")
|
||||
if intentional_diffs:
|
||||
print(
|
||||
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
|
||||
f"no_provider_error → plugin dispatch when a plugin is registered."
|
||||
)
|
||||
if failures or errors:
|
||||
return 1
|
||||
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Behavior-parity check for the TTS plugin hook (issue #30398).
|
||||
|
||||
Spawns one subprocess per (version, scenario) cell — pinned to either
|
||||
``origin/main`` (no plugin hook; ``tts.provider: cartesia`` falls
|
||||
through to the Edge TTS default branch) or this PR's worktree (plugin
|
||||
hook present; same config routes through the plugin registry when a
|
||||
plugin is registered).
|
||||
|
||||
Each subprocess clears all TTS-related env vars + writes a
|
||||
``config.yaml``, then resolves how the dispatcher would route a
|
||||
``text_to_speech`` call. The emitted shape tuple is::
|
||||
|
||||
{dispatch_kind, provider_name, voice_compat}
|
||||
|
||||
Where ``dispatch_kind`` ∈
|
||||
``{"builtin_edge", "builtin_openai", "builtin_elevenlabs", ...,
|
||||
"command", "plugin", "fallback_edge", "error"}``:
|
||||
|
||||
* ``builtin_<name>`` — config selects a built-in handler that exists
|
||||
on both main and PR (no diff expected)
|
||||
* ``command`` — config selects a ``tts.providers.<name>: type: command``
|
||||
entry (PR #17843; no diff expected)
|
||||
* ``plugin`` — config selects a plugin-registered provider (PR only)
|
||||
* ``fallback_edge`` — config selects an unknown name with no matching
|
||||
plugin or command entry → Edge TTS default fallback
|
||||
* ``error`` — explicit fatal error (e.g. mistral quarantine)
|
||||
|
||||
The parent process diffs the reduced shape per scenario. The only
|
||||
acceptable diff is ``fallback_edge → plugin`` for the
|
||||
``unknown-name-with-plugin-installed`` scenario — everything else is
|
||||
a regression.
|
||||
|
||||
Run from the PR worktree (it auto-resolves ``MAIN_DIR`` from the parent
|
||||
of the worktree directory, or falls back to a sibling
|
||||
``hermes-agent-main`` checkout)::
|
||||
|
||||
python tests/plugins/tts/check_parity_vs_main.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _resolve_main_dir() -> Path:
|
||||
candidate = REPO_ROOT.parent.parent
|
||||
if (candidate / "tools" / "tts_tool.py").exists() and candidate != REPO_ROOT:
|
||||
return candidate
|
||||
sibling = REPO_ROOT.parent / "hermes-agent-main"
|
||||
if (sibling / "tools" / "tts_tool.py").exists():
|
||||
return sibling
|
||||
return REPO_ROOT
|
||||
|
||||
|
||||
MAIN_DIR = _resolve_main_dir()
|
||||
PR_DIR = REPO_ROOT
|
||||
assert (PR_DIR / "tools" / "tts_tool.py").exists(), (
|
||||
f"PR_DIR={PR_DIR} doesn't look like a hermes-agent checkout"
|
||||
)
|
||||
|
||||
|
||||
# The subprocess script — runs INSIDE either the main checkout or PR
|
||||
# checkout, so the import paths resolve to the version of the code
|
||||
# under test. We never call the real ``text_to_speech_tool`` because
|
||||
# that would require audio synthesis; instead we ask the resolution
|
||||
# layer what it WOULD do.
|
||||
SUBPROCESS_SCRIPT = r"""
|
||||
import json, os, sys, tempfile
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
|
||||
# Isolated HERMES_HOME so the config write is hermetic.
|
||||
home = tempfile.mkdtemp()
|
||||
os.environ["HERMES_HOME"] = home
|
||||
|
||||
# Clear TTS-related env so dispatch decisions are config-driven.
|
||||
for k in (
|
||||
"ELEVENLABS_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY",
|
||||
"MINIMAX_API_KEY", "XAI_API_KEY", "GEMINI_API_KEY",
|
||||
):
|
||||
os.environ.pop(k, None)
|
||||
|
||||
scenario_env = json.loads(sys.argv[2])
|
||||
os.environ.update(scenario_env)
|
||||
|
||||
config_yaml = sys.argv[3]
|
||||
plugin_register = sys.argv[4] # "yes" to register a fake plugin
|
||||
|
||||
config_path = os.path.join(home, "config.yaml")
|
||||
with open(config_path, "w") as f:
|
||||
f.write(config_yaml)
|
||||
|
||||
# Fresh import — must not have anything cached from prior runs.
|
||||
for name in list(sys.modules):
|
||||
if (name.startswith("tools.")
|
||||
or name.startswith("agent.")
|
||||
or name.startswith("plugins.")
|
||||
or name.startswith("hermes_cli.")):
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
# Try importing tts_registry — only exists on PR side.
|
||||
have_plugin_hook = False
|
||||
try:
|
||||
from agent import tts_registry
|
||||
from agent.tts_provider import TTSProvider
|
||||
have_plugin_hook = True
|
||||
|
||||
if plugin_register == "yes":
|
||||
class _FakeProvider(TTSProvider):
|
||||
@property
|
||||
def name(self): return "cartesia"
|
||||
def synthesize(self, text, output_path, **kw):
|
||||
return output_path
|
||||
|
||||
tts_registry._reset_for_tests()
|
||||
tts_registry.register_provider(_FakeProvider())
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import tools.tts_tool as tts_tool
|
||||
|
||||
# Read the config the same way text_to_speech_tool() does.
|
||||
tts_config = tts_tool._load_tts_config()
|
||||
provider = tts_tool._get_provider(tts_config)
|
||||
|
||||
dispatch_kind = None
|
||||
provider_name = provider
|
||||
voice_compat = False
|
||||
error_text = None
|
||||
|
||||
try:
|
||||
# Mistral is the one branch that returns a fatal error.
|
||||
if provider == "mistral":
|
||||
dispatch_kind = "error"
|
||||
error_text = "mistral quarantine"
|
||||
elif tts_tool._resolve_command_provider_config(provider, tts_config) is not None:
|
||||
dispatch_kind = "command"
|
||||
elif have_plugin_hook and provider not in tts_tool.BUILTIN_TTS_PROVIDERS:
|
||||
# On PR side: check plugin dispatch.
|
||||
plugin_path = tts_tool._dispatch_to_plugin_provider(
|
||||
"test", os.path.join(home, "out.mp3"), provider, tts_config,
|
||||
)
|
||||
if plugin_path is not None:
|
||||
dispatch_kind = "plugin"
|
||||
voice_compat = tts_tool._plugin_provider_is_voice_compatible(provider)
|
||||
else:
|
||||
# Falls through to Edge TTS default on the PR side too.
|
||||
dispatch_kind = "fallback_edge"
|
||||
elif provider in tts_tool.BUILTIN_TTS_PROVIDERS:
|
||||
dispatch_kind = "builtin_" + provider
|
||||
else:
|
||||
# On main side: unknown names fall through to Edge default.
|
||||
dispatch_kind = "fallback_edge"
|
||||
except Exception as exc:
|
||||
dispatch_kind = "exception"
|
||||
error_text = repr(exc)
|
||||
|
||||
shape = {
|
||||
"dispatch_kind": dispatch_kind,
|
||||
"provider_name": provider_name,
|
||||
"voice_compat": bool(voice_compat),
|
||||
"error_present": error_text is not None,
|
||||
}
|
||||
print(json.dumps(shape))
|
||||
"""
|
||||
|
||||
|
||||
SCENARIOS: list[tuple[str, str, dict[str, str], str]] = [
|
||||
# (label, config.yaml body, scenario_env, plugin_register)
|
||||
|
||||
# Scenario 1: unset tts.provider → both: Edge default
|
||||
("unset-defaults-to-edge", "", {}, "no"),
|
||||
|
||||
# Scenario 2: built-in name → both: that built-in
|
||||
("explicit-edge", "tts:\n provider: edge\n", {}, "no"),
|
||||
("explicit-openai", "tts:\n provider: openai\n", {}, "no"),
|
||||
("explicit-elevenlabs", "tts:\n provider: elevenlabs\n", {}, "no"),
|
||||
|
||||
# Scenario 3: command-type provider → both: command dispatch
|
||||
(
|
||||
"command-provider",
|
||||
"tts:\n provider: my-piper\n providers:\n my-piper:\n type: command\n command: 'piper -m model.onnx -f {output_path} < {input_path}'\n",
|
||||
{},
|
||||
"no",
|
||||
),
|
||||
|
||||
# Scenario 4: unknown name with NO plugin installed → both: fallback to Edge
|
||||
("unknown-no-plugin", "tts:\n provider: cartesia\n", {}, "no"),
|
||||
|
||||
# Scenario 5: unknown name WITH plugin installed
|
||||
# main: fallback_edge (no plugin hook exists)
|
||||
# PR: plugin (cartesia)
|
||||
# This is the ONLY acceptable diff in the harness.
|
||||
("plugin-installed", "tts:\n provider: cartesia\n", {}, "yes"),
|
||||
|
||||
# Scenario 6: built-in name + plugin tries to shadow → both: built-in
|
||||
# The plugin registers under name "cartesia", not "edge", so this is
|
||||
# effectively the same as scenario 2 — but we exercise the with-plugin
|
||||
# path to ensure the built-in branch still takes priority.
|
||||
("explicit-edge-with-plugin-registered", "tts:\n provider: edge\n", {}, "yes"),
|
||||
|
||||
# Scenario 7: mistral quarantine — both surface the explicit error
|
||||
("mistral-quarantine", "tts:\n provider: mistral\n", {}, "no"),
|
||||
]
|
||||
|
||||
|
||||
def _run_scenario(repo_path: Path, label: str, config_yaml: str, env: dict, plugin_register: str) -> dict:
|
||||
venv_python = repo_path / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = MAIN_DIR / ".venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = MAIN_DIR / "venv" / "bin" / "python"
|
||||
if not venv_python.exists():
|
||||
venv_python = Path("python3")
|
||||
|
||||
out = subprocess.run(
|
||||
[
|
||||
str(venv_python),
|
||||
"-c",
|
||||
SUBPROCESS_SCRIPT,
|
||||
str(repo_path),
|
||||
json.dumps(env),
|
||||
config_yaml,
|
||||
plugin_register,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return {
|
||||
"error": "subprocess failed",
|
||||
"stdout": out.stdout[-500:],
|
||||
"stderr": out.stderr[-500:],
|
||||
}
|
||||
try:
|
||||
return json.loads(out.stdout.strip().splitlines()[-1])
|
||||
except Exception as exc:
|
||||
return {"error": f"could not parse output: {exc}", "stdout": out.stdout}
|
||||
|
||||
|
||||
def _reduce(shape: dict) -> dict:
|
||||
"""Reduce to the parts that matter for user-visible parity."""
|
||||
return {
|
||||
"dispatch_kind": shape.get("dispatch_kind"),
|
||||
"provider_name": shape.get("provider_name"),
|
||||
"error_present": shape.get("error_present"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print(f"main: {MAIN_DIR}")
|
||||
print(f"pr: {PR_DIR}")
|
||||
print()
|
||||
|
||||
if MAIN_DIR == PR_DIR:
|
||||
print(
|
||||
"WARN: MAIN_DIR == PR_DIR — diffs will be trivially identical.\n"
|
||||
" Set up a sibling 'hermes-agent-main' checkout pinned to "
|
||||
"origin/main to get real parity coverage."
|
||||
)
|
||||
print()
|
||||
|
||||
failures: list[str] = []
|
||||
errors: list[str] = []
|
||||
intentional_diffs: list[tuple[str, dict, dict]] = []
|
||||
for label, config_yaml, env, plugin_register in SCENARIOS:
|
||||
main_shape = _run_scenario(MAIN_DIR, label, config_yaml, env, plugin_register)
|
||||
pr_shape = _run_scenario(PR_DIR, label, config_yaml, env, plugin_register)
|
||||
|
||||
if "error" in main_shape or "error" in pr_shape:
|
||||
print(f" [ERR ] {label}: subprocess failed")
|
||||
print(f" main: {main_shape}")
|
||||
print(f" pr: {pr_shape}")
|
||||
errors.append(label)
|
||||
continue
|
||||
|
||||
main_reduced = _reduce(main_shape)
|
||||
pr_reduced = _reduce(pr_shape)
|
||||
|
||||
if main_reduced == pr_reduced:
|
||||
print(f" [OK] {label}: {main_reduced}")
|
||||
continue
|
||||
|
||||
# On main, "plugin-installed" scenario returns fallback_edge
|
||||
# (no plugin hook); on PR, it routes to the plugin. That's the
|
||||
# only acceptable diff.
|
||||
fallback_to_plugin = (
|
||||
main_reduced.get("dispatch_kind") == "fallback_edge"
|
||||
and pr_reduced.get("dispatch_kind") == "plugin"
|
||||
and label == "plugin-installed"
|
||||
)
|
||||
if fallback_to_plugin:
|
||||
print(f" [DIFF] {label}: fallback_edge → plugin — expected")
|
||||
intentional_diffs.append((label, main_reduced, pr_reduced))
|
||||
else:
|
||||
print(f" [FAIL] {label}")
|
||||
print(f" main: {main_reduced}")
|
||||
print(f" pr: {pr_reduced}")
|
||||
failures.append(label)
|
||||
|
||||
print()
|
||||
if errors:
|
||||
print(f"SUBPROCESS ERRORS in {len(errors)} scenario(s):")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
if failures:
|
||||
print(f"BEHAVIOUR REGRESSION in {len(failures)} scenario(s):")
|
||||
for f in failures:
|
||||
print(f" - {f}")
|
||||
if intentional_diffs:
|
||||
print(
|
||||
f"INTENTIONAL DIFFS ({len(intentional_diffs)}): "
|
||||
f"fallback_edge → plugin dispatch when a plugin is registered."
|
||||
)
|
||||
if failures or errors:
|
||||
return 1
|
||||
print(f"PARITY OK across {len(SCENARIOS)} scenarios.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1 @@
|
||||
"""Make tests/plugins/video_gen a package."""
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Tests for the FAL video gen plugin — family routing, payload shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
def test_fal_provider_registers():
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, DEFAULT_MODEL
|
||||
|
||||
provider = FALVideoGenProvider()
|
||||
video_gen_registry.register_provider(provider)
|
||||
|
||||
assert video_gen_registry.get_provider("fal") is provider
|
||||
assert provider.display_name == "FAL"
|
||||
# DEFAULT_MODEL is the cheap-tier default
|
||||
assert provider.default_model() == DEFAULT_MODEL
|
||||
assert DEFAULT_MODEL in {"pixverse-v6", "ltx-2.3"}
|
||||
|
||||
|
||||
def test_fal_family_catalog():
|
||||
"""Each family declares both endpoints. The catalog covers the
|
||||
cheap + premium tiers Teknium listed."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES
|
||||
|
||||
expected = {
|
||||
# cheap
|
||||
"ltx-2.3", "pixverse-v6",
|
||||
# premium
|
||||
"veo3.1", "seedance-2.0", "kling-v3-4k", "happy-horse",
|
||||
}
|
||||
assert expected.issubset(set(FAL_FAMILIES.keys())), (
|
||||
f"missing families: {expected - set(FAL_FAMILIES.keys())}"
|
||||
)
|
||||
for fid, meta in FAL_FAMILIES.items():
|
||||
assert meta.get("text_endpoint"), f"{fid} missing text_endpoint"
|
||||
assert meta.get("image_endpoint"), f"{fid} missing image_endpoint"
|
||||
assert meta["text_endpoint"] != meta["image_endpoint"]
|
||||
assert meta.get("tier") in {"cheap", "premium"}, (
|
||||
f"{fid} has invalid tier"
|
||||
)
|
||||
|
||||
|
||||
def test_kling_4k_uses_start_image_url():
|
||||
"""Kling v3 4K's image-to-video endpoint expects start_image_url,
|
||||
not image_url. The family must declare image_param_key='start_image_url'."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["kling-v3-4k"]
|
||||
assert meta.get("image_param_key") == "start_image_url"
|
||||
payload = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url="https://example.com/i.png",
|
||||
duration=5,
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt=None,
|
||||
audio=None,
|
||||
seed=None,
|
||||
)
|
||||
assert payload.get("start_image_url") == "https://example.com/i.png"
|
||||
assert "image_url" not in payload
|
||||
|
||||
|
||||
def test_fal_list_models_advertises_both_modalities():
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
models = FALVideoGenProvider().list_models()
|
||||
for m in models:
|
||||
assert set(m["modalities"]) == {"text", "image"}, (
|
||||
f"{m['id']} doesn't advertise both modalities — every family "
|
||||
f"should have t2v + i2v"
|
||||
)
|
||||
|
||||
|
||||
def test_fal_unavailable_without_key(monkeypatch):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
from plugins.video_gen import fal as fal_plugin
|
||||
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
# Also ensure managed gateway is unavailable
|
||||
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
|
||||
assert FALVideoGenProvider().is_available() is False
|
||||
|
||||
|
||||
def test_fal_generate_requires_fal_key(monkeypatch):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
from plugins.video_gen import fal as fal_plugin
|
||||
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
# Also ensure managed gateway is unavailable
|
||||
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
|
||||
result = FALVideoGenProvider().generate("a happy dog")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
|
||||
def test_fal_available_via_gateway(monkeypatch):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
from plugins.video_gen import fal as fal_plugin
|
||||
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
monkeypatch.setattr(
|
||||
fal_plugin,
|
||||
"_resolve_managed_fal_video_gateway",
|
||||
lambda: object(), # truthy sentinel — gateway is available
|
||||
)
|
||||
assert FALVideoGenProvider().is_available() is True
|
||||
|
||||
|
||||
class TestFamilyRouting:
|
||||
"""The headline behavior: image_url presence picks the endpoint."""
|
||||
|
||||
@pytest.fixture
|
||||
def with_fake_fal(self, monkeypatch):
|
||||
"""Stub fal_client.submit to capture which endpoint we hit."""
|
||||
import sys
|
||||
import types
|
||||
|
||||
captured = {"endpoint": None, "arguments": None}
|
||||
|
||||
class FakeHandle:
|
||||
def get(self):
|
||||
return {"video": {"url": "https://fake/out.mp4"}}
|
||||
|
||||
fake = types.ModuleType("fal_client")
|
||||
def _submit(endpoint, arguments=None, headers=None):
|
||||
captured["endpoint"] = endpoint
|
||||
captured["arguments"] = arguments
|
||||
return FakeHandle()
|
||||
fake.submit = _submit # type: ignore
|
||||
monkeypatch.setitem(sys.modules, "fal_client", fake)
|
||||
|
||||
# Reset the lazy global so it picks up our stub
|
||||
from plugins.video_gen import fal as fal_plugin
|
||||
fal_plugin._fal_client = None
|
||||
# Also reset the managed client cache
|
||||
fal_plugin._managed_fal_video_client = None
|
||||
fal_plugin._managed_fal_video_client_config = None
|
||||
|
||||
monkeypatch.setenv("FAL_KEY", "test")
|
||||
# Force direct mode — no managed gateway
|
||||
monkeypatch.setattr(fal_plugin, "_resolve_managed_fal_video_gateway", lambda: None)
|
||||
return captured
|
||||
|
||||
def test_text_to_video_routes_to_text_endpoint(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"a dog running",
|
||||
model="pixverse-v6",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/text-to-video"
|
||||
assert result["modality"] == "text"
|
||||
assert with_fake_fal["arguments"]["prompt"] == "a dog running"
|
||||
assert "image_url" not in with_fake_fal["arguments"]
|
||||
|
||||
def test_image_to_video_routes_to_image_endpoint(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"animate this dog",
|
||||
model="pixverse-v6",
|
||||
image_url="https://example.com/dog.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/image-to-video"
|
||||
assert result["modality"] == "image"
|
||||
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
|
||||
|
||||
def test_default_family_text_routing(self, with_fake_fal):
|
||||
"""No model arg → DEFAULT_MODEL → text-to-video endpoint."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
|
||||
|
||||
result = FALVideoGenProvider().generate("a dog")
|
||||
assert result["success"] is True
|
||||
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
|
||||
assert with_fake_fal["endpoint"] == expected_endpoint
|
||||
|
||||
def test_default_family_image_routing(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"animate this",
|
||||
image_url="https://example.com/i.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["image_endpoint"]
|
||||
assert with_fake_fal["endpoint"] == expected_endpoint
|
||||
|
||||
def test_unknown_family_falls_back_to_default(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"x",
|
||||
model="not-a-real-family",
|
||||
)
|
||||
assert result["success"] is True
|
||||
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
|
||||
assert with_fake_fal["endpoint"] == expected_endpoint
|
||||
|
||||
def test_premium_seedance_routing(self, with_fake_fal):
|
||||
"""Sanity check the premium-tier seedance routes correctly."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"a dog",
|
||||
model="seedance-2.0",
|
||||
image_url="https://example.com/dog.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "bytedance/seedance-2.0/image-to-video"
|
||||
# Seedance uses regular image_url (not start_image_url)
|
||||
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
|
||||
|
||||
def test_kling_4k_remaps_image_param(self, with_fake_fal):
|
||||
"""Kling v3 4K image-to-video receives start_image_url, not image_url."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"x",
|
||||
model="kling-v3-4k",
|
||||
image_url="https://example.com/frame.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "fal-ai/kling-video/v3/4k/image-to-video"
|
||||
assert with_fake_fal["arguments"].get("start_image_url") == "https://example.com/frame.png"
|
||||
assert "image_url" not in with_fake_fal["arguments"]
|
||||
|
||||
|
||||
class TestPayloadBuilder:
|
||||
def test_drops_unsupported_keys(self):
|
||||
"""Veo enum-clamps duration, supports aspect+resolution+audio+neg."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["veo3.1"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url=None,
|
||||
duration=12, # not in enum (4,6,8) — snap to 8
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt="ugly",
|
||||
audio=True,
|
||||
seed=42,
|
||||
)
|
||||
assert p["prompt"] == "x"
|
||||
assert p["duration"] == "8s" # veo3.1 uses "Ns" format per FAL API
|
||||
assert p["aspect_ratio"] == "16:9"
|
||||
assert p["resolution"] == "720p"
|
||||
assert p["generate_audio"] is True
|
||||
assert p["negative_prompt"] == "ugly"
|
||||
assert p["seed"] == 42
|
||||
|
||||
def test_pixverse_range_clamps_correctly(self):
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["pixverse-v6"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url="https://i.png",
|
||||
duration=99, # over max → 15
|
||||
aspect_ratio="16:9",
|
||||
resolution="540p",
|
||||
negative_prompt=None,
|
||||
audio=None,
|
||||
seed=None,
|
||||
)
|
||||
assert p["duration"] == "15"
|
||||
|
||||
def test_kling_4k_clamps_below_min(self):
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["kling-v3-4k"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url="https://i.png",
|
||||
duration=1, # below min (3) → 3
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt=None,
|
||||
audio=None,
|
||||
seed=None,
|
||||
)
|
||||
assert p["duration"] == "3"
|
||||
|
||||
def test_ltx_omits_duration_aspect_resolution(self):
|
||||
"""LTX 2.3 doesn't declare duration/aspect/resolution enums —
|
||||
the payload should NOT include those keys (let FAL default)."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["ltx-2.3"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url=None,
|
||||
duration=8,
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt="ugly",
|
||||
audio=True,
|
||||
seed=None,
|
||||
)
|
||||
assert "duration" not in p
|
||||
assert "aspect_ratio" not in p
|
||||
assert "resolution" not in p
|
||||
# But audio + negative are advertised
|
||||
assert p["generate_audio"] is True
|
||||
assert p["negative_prompt"] == "ugly"
|
||||
|
||||
def test_happy_horse_minimal_payload(self):
|
||||
"""Happy Horse has sparse docs — payload should be minimal."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["happy-horse"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="a horse galloping",
|
||||
image_url=None,
|
||||
duration=8,
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt="watermark",
|
||||
audio=True,
|
||||
seed=None,
|
||||
)
|
||||
# Only prompt — no payload bloat for fields we can't verify
|
||||
assert p == {"prompt": "a horse galloping"}
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Smoke tests for the xAI video gen plugin — load & register surface."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
def test_xai_provider_registers():
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
provider = XAIVideoGenProvider()
|
||||
video_gen_registry.register_provider(provider)
|
||||
|
||||
assert video_gen_registry.get_provider("xai") is provider
|
||||
assert provider.display_name == "xAI"
|
||||
assert provider.default_model() == "grok-imagine-video"
|
||||
|
||||
|
||||
def test_xai_capabilities_text_and_image_only():
|
||||
"""xAI was previously advertised with edit/extend operations. The
|
||||
simplified surface only exposes text-to-video and image-to-video —
|
||||
confirm those are the only modalities advertised."""
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
caps = XAIVideoGenProvider().capabilities()
|
||||
assert caps["modalities"] == ["text", "image"]
|
||||
# No 'operations' key in the simplified surface
|
||||
assert "operations" not in caps
|
||||
assert caps["max_reference_images"] == 7
|
||||
|
||||
|
||||
def test_xai_unavailable_without_key(monkeypatch):
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
assert XAIVideoGenProvider().is_available() is False
|
||||
|
||||
|
||||
def test_xai_generate_requires_xai_key(monkeypatch):
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
result = XAIVideoGenProvider().generate("a happy dog")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
|
||||
def test_xai_available_with_oauth_only(monkeypatch):
|
||||
"""The plugin must honour xAI Grok OAuth credentials, not just
|
||||
XAI_API_KEY. Otherwise the agent's tool-availability check filters
|
||||
``video_generate`` out of the toolbelt and the agent silently falls
|
||||
back to whatever skill advertises video generation (e.g. comfyui).
|
||||
"""
|
||||
import plugins.video_gen.xai as xai_plugin
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(
|
||||
"tools.xai_http.resolve_xai_http_credentials",
|
||||
lambda: {
|
||||
"provider": "xai-oauth",
|
||||
"api_key": "oauth-bearer-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
},
|
||||
)
|
||||
|
||||
assert xai_plugin.XAIVideoGenProvider().is_available() is True
|
||||
|
||||
|
||||
def test_xai_resolved_credentials_threaded_through_request(monkeypatch):
|
||||
"""OAuth-resolved creds must reach the HTTP layer — bug class where
|
||||
``is_available()`` says yes but the request still hits with no key.
|
||||
"""
|
||||
import plugins.video_gen.xai as xai_plugin
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(
|
||||
"tools.xai_http.resolve_xai_http_credentials",
|
||||
lambda: {
|
||||
"provider": "xai-oauth",
|
||||
"api_key": "oauth-bearer-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
},
|
||||
)
|
||||
|
||||
api_key, base_url = xai_plugin._resolve_xai_credentials()
|
||||
assert api_key == "oauth-bearer-token"
|
||||
assert base_url == "https://api.x.ai/v1"
|
||||
headers = xai_plugin._xai_headers(api_key)
|
||||
assert headers["Authorization"] == "Bearer oauth-bearer-token"
|
||||
|
||||
|
||||
def test_xai_no_operation_kwarg():
|
||||
"""The ABC's generate() signature no longer accepts 'operation'.
|
||||
Passing it through **kwargs should be ignored (forward-compat)."""
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
# We're not actually hitting the network — just verify the call
|
||||
# doesn't TypeError on the unexpected kwarg.
|
||||
# Will fail with auth_required (no XAI_API_KEY), but should NOT
|
||||
# fail with TypeError.
|
||||
result = XAIVideoGenProvider().generate("x", operation="generate")
|
||||
assert result["success"] is False
|
||||
# auth_required, NOT some signature error
|
||||
assert result["error_type"] in {"auth_required", "api_error"}
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Integration tests for the xAI video gen plugin's simplified surface.
|
||||
|
||||
xAI exposes only text-to-video and image-to-video through the unified
|
||||
``video_generate`` tool. We assert the endpoint hit and the payload shape
|
||||
because routing is the part most likely to break silently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status: int = 200, payload: Optional[Dict[str, Any]] = None):
|
||||
self.status_code = status
|
||||
self._payload = payload or {}
|
||||
self.text = json.dumps(self._payload)
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
import httpx
|
||||
raise httpx.HTTPStatusError("err", request=None, response=self) # type: ignore
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
def __init__(self):
|
||||
self.posts: List[Dict[str, Any]] = []
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
return None
|
||||
|
||||
async def post(self, url, headers=None, json=None, timeout=None):
|
||||
self.posts.append({"url": url, "json": json})
|
||||
return _FakeResponse(200, {"request_id": "req-123"})
|
||||
|
||||
async def get(self, url, headers=None, timeout=None):
|
||||
return _FakeResponse(200, {
|
||||
"status": "done",
|
||||
"video": {"url": "https://xai-cdn/out.mp4", "duration": 8},
|
||||
"model": "grok-imagine-video",
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def xai_provider(monkeypatch):
|
||||
monkeypatch.setenv("XAI_API_KEY", "test-key")
|
||||
|
||||
import plugins.video_gen.xai as xai_plugin
|
||||
|
||||
captured: Dict[str, _FakeAsyncClient] = {}
|
||||
|
||||
def _client_factory():
|
||||
captured["client"] = _FakeAsyncClient()
|
||||
return captured["client"]
|
||||
|
||||
monkeypatch.setattr(xai_plugin.httpx, "AsyncClient", _client_factory)
|
||||
|
||||
async def _no_sleep(*a, **k):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
|
||||
|
||||
provider = xai_plugin.XAIVideoGenProvider()
|
||||
return provider, captured
|
||||
|
||||
|
||||
def _last_post(captured) -> Dict[str, Any]:
|
||||
return captured["client"].posts[-1]
|
||||
|
||||
|
||||
class TestXAIEndpoint:
|
||||
"""xAI uses one endpoint — ``/videos/generations`` — for both modes."""
|
||||
|
||||
def test_text_to_video_hits_generations(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate("a dog on a skateboard")
|
||||
assert result["success"] is True
|
||||
assert _last_post(captured)["url"].endswith("/videos/generations")
|
||||
assert result["modality"] == "text"
|
||||
|
||||
def test_image_to_video_hits_generations(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate(
|
||||
"animate this",
|
||||
image_url="https://example.com/cat.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert _last_post(captured)["url"].endswith("/videos/generations")
|
||||
assert result["modality"] == "image"
|
||||
|
||||
|
||||
class TestXAIPayload:
|
||||
def test_text_payload_has_no_image_field(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("a dog at sunset")
|
||||
payload = _last_post(captured)["json"]
|
||||
assert payload["prompt"] == "a dog at sunset"
|
||||
assert "image" not in payload
|
||||
assert "reference_images" not in payload
|
||||
|
||||
def test_image_payload_has_image_field(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("animate this", image_url="https://example.com/cat.png")
|
||||
payload = _last_post(captured)["json"]
|
||||
assert payload["image"] == {"url": "https://example.com/cat.png"}
|
||||
|
||||
def test_reference_images_payload(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate(
|
||||
"keep this character",
|
||||
reference_image_urls=[
|
||||
"https://example.com/a.png",
|
||||
"https://example.com/b.png",
|
||||
],
|
||||
)
|
||||
payload = _last_post(captured)["json"]
|
||||
assert payload["reference_images"] == [
|
||||
{"url": "https://example.com/a.png"},
|
||||
{"url": "https://example.com/b.png"},
|
||||
]
|
||||
|
||||
|
||||
class TestXAIValidation:
|
||||
def test_missing_prompt_rejects(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate("")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "missing_prompt"
|
||||
# Never hit the network
|
||||
assert "client" not in captured or not captured["client"].posts
|
||||
|
||||
def test_image_plus_refs_rejects(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate(
|
||||
"x",
|
||||
image_url="https://example.com/i.png",
|
||||
reference_image_urls=["https://example.com/r.png"],
|
||||
)
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "conflicting_inputs"
|
||||
assert "client" not in captured or not captured["client"].posts
|
||||
|
||||
def test_too_many_references_rejects(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate(
|
||||
"x",
|
||||
reference_image_urls=[f"https://example.com/r{i}.png" for i in range(8)],
|
||||
)
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "too_many_references"
|
||||
|
||||
|
||||
class TestXAIClamping:
|
||||
def test_duration_clamped_to_15(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("x", duration=30)
|
||||
assert _last_post(captured)["json"]["duration"] == 15
|
||||
|
||||
def test_duration_clamped_when_refs_present(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate(
|
||||
"x",
|
||||
duration=15,
|
||||
reference_image_urls=["https://example.com/r.png"],
|
||||
)
|
||||
# refs present caps to 10
|
||||
assert _last_post(captured)["json"]["duration"] == 10
|
||||
|
||||
def test_invalid_aspect_ratio_soft_clamps(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("x", aspect_ratio="21:9")
|
||||
assert _last_post(captured)["json"]["aspect_ratio"] == "16:9"
|
||||
@@ -0,0 +1,498 @@
|
||||
"""Plugin-side tests for the web search provider migration (PR #25182).
|
||||
|
||||
Covers:
|
||||
|
||||
- All eight bundled plugins (brave-free, ddgs, searxng, exa, parallel,
|
||||
tavily, firecrawl, xai) instantiate and self-report the expected
|
||||
capabilities + ABC-derived defaults.
|
||||
- Each plugin's ``is_available()`` correctly reflects env-var presence.
|
||||
- The web_search_registry resolves an active provider in the documented
|
||||
scenarios (explicit config wins ignoring availability, fallback walks
|
||||
legacy preference filtered by availability, unknown name falls back).
|
||||
- Plugin response shapes match the legacy bit-for-bit contract.
|
||||
|
||||
Per the dev skill: these tests use *real* imports from the plugin
|
||||
modules — no mocking of provider classes themselves — so the test
|
||||
catches drift in the ABC interface, the registry, and the plugin
|
||||
glue layer simultaneously.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Strip every web-provider env var so is_available() returns False."""
|
||||
for k in (
|
||||
"BRAVE_SEARCH_API_KEY",
|
||||
"SEARXNG_URL",
|
||||
"TAVILY_API_KEY",
|
||||
"TAVILY_BASE_URL",
|
||||
"EXA_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"PARALLEL_SEARCH_MODE",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
"FIRECRAWL_GATEWAY_URL",
|
||||
"TOOL_GATEWAY_DOMAIN",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
"XAI_API_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
|
||||
def _ensure_plugins_loaded() -> None:
|
||||
"""Idempotently load plugins so the registry is populated."""
|
||||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||||
|
||||
_ensure_plugins_discovered()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-plugin discovery + capability flags
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Each test starts with a clean web-provider env."""
|
||||
_clear_web_env(monkeypatch)
|
||||
|
||||
|
||||
class TestBundledPluginsRegister:
|
||||
"""All eight bundled web plugins discover and register correctly."""
|
||||
|
||||
def test_all_seven_plugins_present_in_registry(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import list_providers
|
||||
|
||||
names = sorted(p.name for p in list_providers())
|
||||
assert names == [
|
||||
"brave-free",
|
||||
"ddgs",
|
||||
"exa",
|
||||
"firecrawl",
|
||||
"parallel",
|
||||
"searxng",
|
||||
"tavily",
|
||||
"xai",
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name,expected_search,expected_extract",
|
||||
[
|
||||
("brave-free", True, False),
|
||||
("ddgs", True, False),
|
||||
("searxng", True, False),
|
||||
("exa", True, True),
|
||||
("parallel", True, True),
|
||||
("tavily", True, True),
|
||||
("firecrawl", True, True),
|
||||
# xai: search-only via Grok's agentic web_search tool.
|
||||
("xai", True, False),
|
||||
],
|
||||
)
|
||||
def test_capability_flags_match_spec(
|
||||
self,
|
||||
plugin_name: str,
|
||||
expected_search: bool,
|
||||
expected_extract: bool,
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None, f"plugin {plugin_name!r} not registered"
|
||||
assert provider.supports_search() is expected_search
|
||||
assert provider.supports_extract() is expected_extract
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"],
|
||||
)
|
||||
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
assert provider.name == plugin_name
|
||||
assert provider.display_name # any non-empty string
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"],
|
||||
)
|
||||
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
|
||||
"""``get_setup_schema()`` returns a dict the picker can consume."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
schema = provider.get_setup_schema()
|
||||
assert isinstance(schema, dict)
|
||||
assert "name" in schema
|
||||
assert "env_vars" in schema
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_available() behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsAvailable:
|
||||
"""Each plugin's ``is_available()`` returns False without env config."""
|
||||
|
||||
def test_brave_free_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("brave-free")
|
||||
assert p is not None
|
||||
assert p.is_available() is False # no BRAVE_SEARCH_API_KEY
|
||||
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_searxng_requires_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("searxng")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("TAVILY_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("exa")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("EXA_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_parallel_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("parallel")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("PARALLEL_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_firecrawl_requires_either_key_or_url(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
|
||||
# Either FIRECRAWL_API_KEY or FIRECRAWL_API_URL lights it up.
|
||||
monkeypatch.setenv("FIRECRAWL_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
|
||||
monkeypatch.setenv("FIRECRAWL_API_URL", "http://localhost:3002")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_ddgs_always_available_when_package_importable(self) -> None:
|
||||
"""DDGS is the always-on fallback — no API key required.
|
||||
|
||||
It may report unavailable if the ``ddgs`` package itself isn't
|
||||
installed in the env (legitimate — the plugin's post_setup hook
|
||||
triggers pip install on first selection). We only assert that
|
||||
is_available() doesn't raise.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("ddgs")
|
||||
assert p is not None
|
||||
# Truthy or falsy, just must not raise.
|
||||
_ = bool(p.is_available())
|
||||
|
||||
def test_xai_requires_api_key_or_oauth(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""xAI needs XAI_API_KEY or OAuth tokens in auth.json."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("xai")
|
||||
assert p is not None
|
||||
assert p.is_available() is False # no XAI_API_KEY, no auth.json
|
||||
monkeypatch.setenv("XAI_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry resolution semantics (Option B — conservative smart fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryResolution:
|
||||
"""``_resolve()`` follows explicit-config + availability-filtered fallback."""
|
||||
|
||||
def test_explicit_configured_provider_returned_even_when_unavailable(
|
||||
self,
|
||||
) -> None:
|
||||
"""Explicit ``web.search_backend`` wins regardless of is_available().
|
||||
|
||||
Without availability filtering on the explicit path, the dispatcher
|
||||
would silently switch backends; with this check the dispatcher
|
||||
surfaces a precise "FOO_API_KEY is not set" error instead.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
# No BRAVE_SEARCH_API_KEY (fixture cleared it).
|
||||
result = _resolve("brave-free", capability="search")
|
||||
assert result is not None
|
||||
assert result.name == "brave-free"
|
||||
# Confirm it's the unavailable one — dispatcher will surface
|
||||
# a typed credential-missing error to the caller.
|
||||
assert result.is_available() is False
|
||||
|
||||
def test_unknown_configured_name_falls_back_to_available_provider(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Typo / uninstalled plugin → walk legacy preference, pick available."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
monkeypatch.setenv("EXA_API_KEY", "real")
|
||||
result = _resolve("not-a-real-provider", capability="search")
|
||||
# Either ddgs (no-key fallback) or exa (the only available
|
||||
# premium provider) — both are valid. The point is the unknown
|
||||
# name shouldn't return None when SOMETHING is available.
|
||||
assert result is not None
|
||||
assert result.is_available() is True
|
||||
|
||||
def test_explicit_search_only_provider_for_extract_falls_back(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Asking for extract via a search-only backend → fall back.
|
||||
|
||||
``brave-free`` is search-only (``supports_extract() is False``).
|
||||
When the registry resolves it for an extract capability, the
|
||||
explicit-config branch rejects it as capability-incompatible
|
||||
and the fallback walk picks an extract-capable provider.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
monkeypatch.setenv("EXA_API_KEY", "real")
|
||||
result = _resolve("brave-free", capability="extract")
|
||||
# Should land on exa (only extract-capable available provider).
|
||||
assert result is not None
|
||||
assert result.supports_extract() is True
|
||||
assert result.is_available() is True
|
||||
|
||||
def test_no_config_no_credentials_returns_none(
|
||||
self,
|
||||
) -> None:
|
||||
"""No backend configured AND no available providers → typically None.
|
||||
|
||||
``ddgs`` is the no-credential fallback; if its ``ddgs`` Python
|
||||
package is installed in the test env, ddgs will be picked.
|
||||
Otherwise the resolver returns None. Either outcome is correct.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
result = _resolve(None, capability="search")
|
||||
if result is not None:
|
||||
# The only no-credential provider is ddgs; anything else
|
||||
# means an env var leaked in.
|
||||
assert result.is_available() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync-vs-async extract detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAsyncExtractDispatch:
|
||||
"""The dispatcher detects async vs sync extract methods correctly."""
|
||||
|
||||
def test_parallel_extract_is_async(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("parallel")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is True
|
||||
|
||||
def test_firecrawl_extract_is_async(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is True
|
||||
|
||||
def test_exa_extract_is_sync(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("exa")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is False
|
||||
|
||||
def test_tavily_extract_is_sync(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error response shape (preserved bit-for-bit from legacy)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorResponseShapes:
|
||||
"""When credentials are missing, plugins return typed errors, not raises."""
|
||||
|
||||
def test_brave_free_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("brave-free")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_searxng_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("searxng")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_exa_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("exa")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_tavily_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_parallel_extract_returns_per_url_errors_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("parallel")
|
||||
assert p is not None
|
||||
result = asyncio.run(p.extract(["https://example.com"]))
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 1
|
||||
assert "error" in result[0]
|
||||
assert result[0]["url"] == "https://example.com"
|
||||
|
||||
def test_firecrawl_extract_returns_per_url_errors_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
# firecrawl extract returns [] when the website-policy gate rejects
|
||||
# the URL, or a per-URL error dict when the gate passes but the
|
||||
# firecrawl client fails. Use a URL the policy allows to make sure
|
||||
# we hit the credential-missing path.
|
||||
result = asyncio.run(p.extract(["https://example.com"]))
|
||||
assert isinstance(result, list)
|
||||
if result: # if anything came back, it should be an error entry
|
||||
assert "error" in result[0]
|
||||
|
||||
def test_firecrawl_config_error_points_paid_users_to_nous_subscription(self, monkeypatch):
|
||||
from plugins.web.firecrawl import provider as firecrawl_provider
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.web_tools.managed_nous_tools_enabled",
|
||||
lambda: True,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
firecrawl_provider._raise_web_backend_configuration_error()
|
||||
|
||||
message = str(exc_info.value)
|
||||
assert "With your Nous subscription you can also use the Tool Gateway" in message
|
||||
assert "select Nous Subscription as the web provider" in message
|
||||
assert "managed Firecrawl web tools is unavailable" not in message
|
||||
|
||||
def test_firecrawl_config_error_uses_entitlement_message_when_not_paid(self, monkeypatch):
|
||||
from plugins.web.firecrawl import provider as firecrawl_provider
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.web_tools.managed_nous_tools_enabled",
|
||||
lambda: False,
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"tools.web_tools.nous_tool_gateway_unavailable_message",
|
||||
lambda capability: f"{capability} denied by test entitlement.",
|
||||
raising=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
firecrawl_provider._raise_web_backend_configuration_error()
|
||||
|
||||
assert "managed Firecrawl web tools denied by test entitlement" in str(exc_info.value)
|
||||
|
||||
def test_xai_search_returns_error_dict_when_unconfigured(self) -> None:
|
||||
"""xAI returns a typed error dict (no XAI_API_KEY)."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("xai")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
Reference in New Issue
Block a user