Add local search quality evaluation harness

Add an optional local evaluator that compares a baseline revision against a candidate checkout, computes deterministic stability metrics, and can call Gemini for judged ranking metrics when configured.

The harness isolates child runs with a temporary HOME and a node-free PATH so historical revisions cannot trigger Bird browser-cookie auth during evaluation.

Validation: uv run python -m unittest and local smoke/full deterministic eval runs.
This commit is contained in:
Jeffrey Sperling
2026-03-13 19:21:33 -07:00
parent 946af84f9a
commit 8eda5fad5c
5 changed files with 732 additions and 0 deletions
+17
View File
@@ -79,6 +79,23 @@ class TestConfigPrecedence(unittest.TestCase):
config = env.get_config()
self.assertEqual(config['BRAVE_API_KEY'], 'env-key')
def test_gemini_keys_load_from_project_env(self):
import tempfile
with tempfile.TemporaryDirectory() as tmpdir:
project_dir = Path(tmpdir) / ".claude"
project_dir.mkdir()
project_env = project_dir / "last30days.env"
project_env.write_text("GEMINI_API_KEY=gem-key\nGEMINI_MODEL=gemini-3-pro-preview\n")
with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
patch.object(env, 'CONFIG_FILE', None), \
patch.dict(os.environ, {}, clear=False):
os.environ.pop('GEMINI_API_KEY', None)
os.environ.pop('GEMINI_MODEL', None)
config = env.get_config()
self.assertEqual(config['GEMINI_API_KEY'], 'gem-key')
self.assertEqual(config['GEMINI_MODEL'], 'gemini-3-pro-preview')
class TestConfigSource(unittest.TestCase):
"""Tests for _CONFIG_SOURCE tracking."""