From 588cff3e0057011382e55ab5149d160571011e07 Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 17:57:15 -0700 Subject: [PATCH] Optimize model selection for cost-efficiency on structured extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task profile is search tool invocation + JSON extraction — not reasoning or creative work. Mini models handle this equally well at 3-5x lower cost per call. OpenAI changes: - Rename is_mainline_openai_model -> is_search_capable_model - Include mini variants (gpt-5-mini, gpt-4.1-mini) in candidate pool - Exclude gpt-4o-mini (no domain filtering) and nano (no web_search) - select_openai_model() now prefers mini within newest generation - OPENAI_FALLBACK_MODELS: gpt-5-mini first, mainline as last resort - MODEL_FALLBACK_ORDER: same mini-first ordering xAI changes: - Switch alias from grok-4-1-fast (reasoning) to grok-4-1-fast-non-reasoning — same token price, faster response, no wasted reasoning tokens for structured extraction Cost per Reddit search call: ~$0.015 (gpt-5-mini) vs ~$0.044 (gpt-4.1) --- scripts/lib/models.py | 75 +++++++++++++++------ scripts/lib/openai_reddit.py | 7 +- tests/test_models.py | 124 +++++++++++++++++++++++++++-------- tests/test_openai_reddit.py | 15 +++-- 4 files changed, 164 insertions(+), 57 deletions(-) diff --git a/scripts/lib/models.py b/scripts/lib/models.py index 2d5ec7f..8888467 100644 --- a/scripts/lib/models.py +++ b/scripts/lib/models.py @@ -1,4 +1,22 @@ -"""Model auto-selection for last30days skill.""" +"""Model auto-selection for last30days skill. + +Model selection philosophy: this tool uses LLM APIs exclusively for +search tool invocation + structured JSON extraction. This is not +reasoning-heavy or creative work — mini models handle it equally well +at ~3-5x lower cost. We prefer the newest-generation mini model, falling +back to mainline only when mini isn't available. + +OpenAI cost per Reddit search call (web_search tool + JSON output): + gpt-4.1-mini: ~$0.014 (fixed 8K search token block) + gpt-5-mini: ~$0.015 + gpt-4.1: ~$0.044 + gpt-5.2: ~$0.043 + gpt-4o: ~$0.053 + +xAI: grok-4-1-fast reasoning vs non-reasoning have identical token +pricing ($0.20/1M in, $0.50/1M out). Non-reasoning skips the thinking +phase, saving latency and reasoning token output costs. +""" import re from typing import Dict, List, Optional, Tuple @@ -7,13 +25,17 @@ from . import cache, http, env # OpenAI API OPENAI_MODELS_URL = "https://api.openai.com/v1/models" -OPENAI_FALLBACK_MODELS = ["gpt-5.2", "gpt-5.1", "gpt-5", "gpt-4.1", "gpt-4o"] +# Ordered by cost-efficiency for web_search + JSON extraction tasks. +# Mini models first: same structured extraction quality at ~3x lower cost. +OPENAI_FALLBACK_MODELS = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1", "gpt-4o"] CODEX_FALLBACK_MODELS = ["gpt-5.1-codex-mini", "gpt-5.2"] # xAI API - Agent Tools API requires grok-4 family +# Non-reasoning: same price, faster, no unnecessary thinking tokens. +# Both variants support function calling and structured outputs. XAI_MODELS_URL = "https://api.x.ai/v1/models" XAI_ALIASES = { - "latest": "grok-4-1-fast-non-reasoning", # Explicit: bare grok-4-1-fast aliases to reasoning variant + "latest": "grok-4-1-fast-non-reasoning", "stable": "grok-4-1-fast-non-reasoning", } @@ -32,30 +54,45 @@ def parse_version(model_id: str) -> Optional[Tuple[int, ...]]: return None -def is_mainline_openai_model(model_id: str) -> bool: - """Check if model is a mainline GPT model (not mini/nano/chat/codex/pro).""" +def is_search_capable_model(model_id: str) -> bool: + """Check if model supports Responses API web_search with domain filtering. + + Includes mini variants (same structured extraction quality, lower cost). + Excludes: nano (no web_search), gpt-4o-mini (no domain filtering), + chat/codex/pro/preview/turbo/search (specialized variants). + """ model_lower = model_id.lower() - # Must be gpt-4o, gpt-4.1+, or gpt-5+ series (mainline, not mini/nano/etc) - if not re.match(r'^gpt-(?:4o|4\.1|5)(\.\d+)*$', model_lower): + # gpt-4o-mini does NOT support web_search with filters — exclude it + if model_lower.startswith("gpt-4o-mini"): return False - # Exclude variants - excludes = ['mini', 'nano', 'chat', 'codex', 'pro', 'preview', 'turbo'] - for exc in excludes: + # Must be gpt-4o, gpt-4.1[-mini], or gpt-5[-mini] series + if not re.match(r'^gpt-(?:4o|4\.1|5)(\.\d+)*(-mini)?$', model_lower): + return False + + # Exclude unsupported variants + for exc in ['nano', 'chat', 'codex', 'pro', 'preview', 'turbo', 'search']: if exc in model_lower: return False return True +# Backward compat alias +is_mainline_openai_model = is_search_capable_model + + def select_openai_model( api_key: str, policy: str = "auto", pin: Optional[str] = None, mock_models: Optional[List[Dict]] = None, ) -> str: - """Select the best OpenAI model based on policy. + """Select the most cost-efficient OpenAI model for web_search + JSON extraction. + + Prefers mini models within the newest generation available, since the task + is structured extraction (not reasoning or creative work). Args: api_key: OpenAI API key @@ -83,26 +120,24 @@ def select_openai_model( response = http.get(OPENAI_MODELS_URL, headers=headers) models = response.get("data", []) except http.HTTPError: - # Fall back to known models return OPENAI_FALLBACK_MODELS[0] - # Filter to mainline models - candidates = [m for m in models if is_mainline_openai_model(m.get("id", ""))] + candidates = [m for m in models if is_search_capable_model(m.get("id", ""))] if not candidates: - # No gpt-5 models found, use fallback return OPENAI_FALLBACK_MODELS[0] - # Sort by version (descending), then by created timestamp + # Sort: newest generation first, prefer mini within same generation def sort_key(m): - version = parse_version(m.get("id", "")) or (0,) - created = m.get("created", 0) - return (version, created) + model_id = m.get("id", "") + version = parse_version(model_id) or (0,) + major = version[0] if version else 0 + is_mini = 1 if "mini" in model_id.lower() else 0 + return (major, is_mini, version) candidates.sort(key=sort_key, reverse=True) selected = candidates[0]["id"] - # Cache the selection cache.set_cached_model("openai", selected) return selected diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py index f98e8e4..afe4b61 100644 --- a/scripts/lib/openai_reddit.py +++ b/scripts/lib/openai_reddit.py @@ -7,9 +7,10 @@ from typing import Any, Dict, List, Optional from . import http, env -# Fallback models when the selected model isn't accessible (e.g., org not verified for GPT-5) -# Note: gpt-4o-mini does NOT support web_search with filters param, so exclude it -MODEL_FALLBACK_ORDER = ["gpt-4.1", "gpt-4o"] +# Fallback models when the selected model isn't accessible (e.g., org not verified). +# Ordered by cost-efficiency: mini models handle structured extraction equally well. +# Note: gpt-4o-mini does NOT support web_search with filters — excluded. +MODEL_FALLBACK_ORDER = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1", "gpt-4o"] def _log_error(msg: str): diff --git a/tests/test_models.py b/tests/test_models.py index 493a4d7..36db570 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -28,21 +28,47 @@ class TestParseVersion(unittest.TestCase): self.assertIsNone(result) -class TestIsMainlineOpenAIModel(unittest.TestCase): - def test_gpt5_is_mainline(self): +class TestIsSearchCapableModel(unittest.TestCase): + def test_gpt5_is_capable(self): + self.assertTrue(models.is_search_capable_model("gpt-5")) + + def test_gpt52_is_capable(self): + self.assertTrue(models.is_search_capable_model("gpt-5.2")) + + def test_gpt5_mini_is_capable(self): + self.assertTrue(models.is_search_capable_model("gpt-5-mini")) + + def test_gpt41_mini_is_capable(self): + self.assertTrue(models.is_search_capable_model("gpt-4.1-mini")) + + def test_gpt4o_is_capable(self): + self.assertTrue(models.is_search_capable_model("gpt-4o")) + + def test_gpt4o_mini_not_capable(self): + """gpt-4o-mini does not support web_search with domain filtering.""" + self.assertFalse(models.is_search_capable_model("gpt-4o-mini")) + + def test_nano_not_capable(self): + """nano models don't support web_search.""" + self.assertFalse(models.is_search_capable_model("gpt-4.1-nano")) + self.assertFalse(models.is_search_capable_model("gpt-5-nano")) + + def test_gpt4_not_capable(self): + self.assertFalse(models.is_search_capable_model("gpt-4")) + + def test_codex_not_capable(self): + self.assertFalse(models.is_search_capable_model("gpt-5.1-codex")) + + def test_backward_compat_alias(self): + """is_mainline_openai_model still works as alias.""" self.assertTrue(models.is_mainline_openai_model("gpt-5")) - def test_gpt52_is_mainline(self): - self.assertTrue(models.is_mainline_openai_model("gpt-5.2")) - - def test_gpt5_mini_is_not_mainline(self): - self.assertFalse(models.is_mainline_openai_model("gpt-5-mini")) - - def test_gpt4_is_not_mainline(self): - self.assertFalse(models.is_mainline_openai_model("gpt-4")) - class TestSelectOpenAIModel(unittest.TestCase): + def setUp(self): + from lib import cache + cache.MODEL_CACHE_FILE.unlink(missing_ok=True) + def test_pinned_policy(self): result = models.select_openai_model( "fake-key", @@ -51,20 +77,8 @@ class TestSelectOpenAIModel(unittest.TestCase): ) self.assertEqual(result, "gpt-5.1") - def test_auto_with_mock_models(self): - mock_models = [ - {"id": "gpt-5.2", "created": 1704067200}, - {"id": "gpt-5.1", "created": 1701388800}, - {"id": "gpt-5", "created": 1698710400}, - ] - result = models.select_openai_model( - "fake-key", - policy="auto", - mock_models=mock_models - ) - self.assertEqual(result, "gpt-5.2") - - def test_auto_filters_variants(self): + def test_prefers_mini_over_mainline(self): + """Mini models should be preferred for cost-efficiency.""" mock_models = [ {"id": "gpt-5.2", "created": 1704067200}, {"id": "gpt-5-mini", "created": 1704067200}, @@ -75,8 +89,50 @@ class TestSelectOpenAIModel(unittest.TestCase): policy="auto", mock_models=mock_models ) + self.assertEqual(result, "gpt-5-mini") + + def test_prefers_newer_generation_mini(self): + """gpt-5-mini should beat gpt-4.1-mini (newer generation).""" + mock_models = [ + {"id": "gpt-4.1-mini", "created": 1701388800}, + {"id": "gpt-5-mini", "created": 1704067200}, + {"id": "gpt-4.1", "created": 1698710400}, + ] + result = models.select_openai_model( + "fake-key", + policy="auto", + mock_models=mock_models + ) + self.assertEqual(result, "gpt-5-mini") + + def test_falls_back_to_mainline_when_no_mini(self): + """Without mini models, mainline models are selected.""" + mock_models = [ + {"id": "gpt-5.2", "created": 1704067200}, + {"id": "gpt-4.1", "created": 1698710400}, + ] + result = models.select_openai_model( + "fake-key", + policy="auto", + mock_models=mock_models + ) self.assertEqual(result, "gpt-5.2") + def test_filters_unsupported_variants(self): + """Nano, codex, preview models should be excluded.""" + mock_models = [ + {"id": "gpt-5-nano", "created": 1704067200}, + {"id": "gpt-5.1-codex", "created": 1704067200}, + {"id": "gpt-4o-mini", "created": 1704067200}, + {"id": "gpt-4.1-mini", "created": 1698710400}, + ] + result = models.select_openai_model( + "fake-key", + policy="auto", + mock_models=mock_models + ) + self.assertEqual(result, "gpt-4.1-mini") + class TestSelectXAIModel(unittest.TestCase): def test_latest_policy(self): @@ -106,6 +162,10 @@ class TestSelectXAIModel(unittest.TestCase): class TestGetModels(unittest.TestCase): + def setUp(self): + from lib import cache + cache.MODEL_CACHE_FILE.unlink(missing_ok=True) + def test_no_keys_returns_none(self): config = {} result = models.get_models(config) @@ -114,9 +174,12 @@ class TestGetModels(unittest.TestCase): def test_openai_key_only(self): config = {"OPENAI_API_KEY": "sk-test"} - mock_models = [{"id": "gpt-5.2", "created": 1704067200}] + mock_models = [ + {"id": "gpt-5.2", "created": 1704067200}, + {"id": "gpt-5-mini", "created": 1704067200}, + ] result = models.get_models(config, mock_openai_models=mock_models) - self.assertEqual(result["openai"], "gpt-5.2") + self.assertEqual(result["openai"], "gpt-5-mini") self.assertIsNone(result["xai"]) def test_both_keys(self): @@ -124,10 +187,13 @@ class TestGetModels(unittest.TestCase): "OPENAI_API_KEY": "sk-test", "XAI_API_KEY": "xai-test", } - mock_openai = [{"id": "gpt-5.2", "created": 1704067200}] + mock_openai = [ + {"id": "gpt-5.2", "created": 1704067200}, + {"id": "gpt-5-mini", "created": 1704067200}, + ] mock_xai = [{"id": "grok-4-1-fast-non-reasoning", "created": 1704067200}] result = models.get_models(config, mock_openai, mock_xai) - self.assertEqual(result["openai"], "gpt-5.2") + self.assertEqual(result["openai"], "gpt-5-mini") self.assertEqual(result["xai"], "grok-4-1-fast-non-reasoning") diff --git a/tests/test_openai_reddit.py b/tests/test_openai_reddit.py index 607be97..74cc4dc 100644 --- a/tests/test_openai_reddit.py +++ b/tests/test_openai_reddit.py @@ -64,13 +64,18 @@ class TestIsModelAccessError(unittest.TestCase): class TestModelFallbackOrder(unittest.TestCase): """Tests for MODEL_FALLBACK_ORDER constant.""" - def test_contains_gpt4o(self): - """Fallback list should include gpt-4o.""" + def test_mini_first(self): + """Mini models should come first (cost-efficient for structured extraction).""" + self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-5-mini") + + def test_contains_mainline_fallbacks(self): + """Fallback list should include mainline models as last resort.""" + self.assertIn("gpt-4.1", MODEL_FALLBACK_ORDER) self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER) - def test_gpt41_is_first(self): - """gpt-4.1 should be the first fallback option.""" - self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4.1") + def test_no_gpt4o_mini(self): + """gpt-4o-mini should NOT be in fallback (no domain filtering support).""" + self.assertNotIn("gpt-4o-mini", MODEL_FALLBACK_ORDER) if __name__ == "__main__":