Jeffrey Sperling
588cff3e00
The task profile is search tool invocation + JSON extraction — not reasoning or creative work. Mini models handle this equally well at 3-5x lower cost per call. OpenAI changes: - Rename is_mainline_openai_model -> is_search_capable_model - Include mini variants (gpt-5-mini, gpt-4.1-mini) in candidate pool - Exclude gpt-4o-mini (no domain filtering) and nano (no web_search) - select_openai_model() now prefers mini within newest generation - OPENAI_FALLBACK_MODELS: gpt-5-mini first, mainline as last resort - MODEL_FALLBACK_ORDER: same mini-first ordering xAI changes: - Switch alias from grok-4-1-fast (reasoning) to grok-4-1-fast-non-reasoning — same token price, faster response, no wasted reasoning tokens for structured extraction Cost per Reddit search call: ~$0.015 (gpt-5-mini) vs ~$0.044 (gpt-4.1)
83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
"""Tests for openai_reddit module."""
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
# Add scripts directory to path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
|
|
|
from lib import http
|
|
from lib.openai_reddit import _is_model_access_error, MODEL_FALLBACK_ORDER
|
|
|
|
|
|
class TestIsModelAccessError(unittest.TestCase):
|
|
"""Tests for _is_model_access_error function."""
|
|
|
|
def test_returns_false_for_non_400_error(self):
|
|
"""Non-400 errors should not trigger fallback."""
|
|
error = http.HTTPError("Server error", status_code=500, body="Internal error")
|
|
self.assertFalse(_is_model_access_error(error))
|
|
|
|
def test_returns_false_for_400_without_body(self):
|
|
"""400 without body should not trigger fallback."""
|
|
error = http.HTTPError("Bad request", status_code=400, body=None)
|
|
self.assertFalse(_is_model_access_error(error))
|
|
|
|
def test_returns_true_for_verification_error(self):
|
|
"""Verification error should trigger fallback."""
|
|
error = http.HTTPError(
|
|
"Bad request",
|
|
status_code=400,
|
|
body='{"error": {"message": "Your organization must be verified to use the model \'gpt-5.2\'"}}'
|
|
)
|
|
self.assertTrue(_is_model_access_error(error))
|
|
|
|
def test_returns_true_for_access_error(self):
|
|
"""Access denied error should trigger fallback."""
|
|
error = http.HTTPError(
|
|
"Bad request",
|
|
status_code=400,
|
|
body='{"error": {"message": "Your account does not have access to this model"}}'
|
|
)
|
|
self.assertTrue(_is_model_access_error(error))
|
|
|
|
def test_returns_true_for_model_not_found(self):
|
|
"""Model not found error should trigger fallback."""
|
|
error = http.HTTPError(
|
|
"Bad request",
|
|
status_code=400,
|
|
body='{"error": {"message": "The model gpt-5.2 was not found"}}'
|
|
)
|
|
self.assertTrue(_is_model_access_error(error))
|
|
|
|
def test_returns_false_for_unrelated_400(self):
|
|
"""Unrelated 400 errors should not trigger fallback."""
|
|
error = http.HTTPError(
|
|
"Bad request",
|
|
status_code=400,
|
|
body='{"error": {"message": "Invalid JSON in request body"}}'
|
|
)
|
|
self.assertFalse(_is_model_access_error(error))
|
|
|
|
|
|
class TestModelFallbackOrder(unittest.TestCase):
|
|
"""Tests for MODEL_FALLBACK_ORDER constant."""
|
|
|
|
def test_mini_first(self):
|
|
"""Mini models should come first (cost-efficient for structured extraction)."""
|
|
self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-5-mini")
|
|
|
|
def test_contains_mainline_fallbacks(self):
|
|
"""Fallback list should include mainline models as last resort."""
|
|
self.assertIn("gpt-4.1", MODEL_FALLBACK_ORDER)
|
|
self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER)
|
|
|
|
def test_no_gpt4o_mini(self):
|
|
"""gpt-4o-mini should NOT be in fallback (no domain filtering support)."""
|
|
self.assertNotIn("gpt-4o-mini", MODEL_FALLBACK_ORDER)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|