fix: YAML frontmatter and GPT-5 model fallback

- Fix invalid YAML in SKILL.md argument-hint (closes #8)
  Wrapped value in single quotes to properly escape double quotes

- Add automatic model fallback for GPT-5 access errors (closes #9)
  When OpenAI returns 400 for unverified orgs, retry with gpt-4o

- Add tests for model fallback logic

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-01-28 11:07:33 -08:00
parent 3a4a727f4b
commit 6cc7dad86e
3 changed files with 146 additions and 21 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
---
name: last30days
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
argument-hint: "[topic] for [tool]" or "[topic]"
argument-hint: '"[topic] for [tool]" or "[topic]"'
context: fork
agent: Explore
disable-model-invocation: true
+68 -20
View File
@@ -7,12 +7,39 @@ from typing import Any, Dict, List, Optional
from . import http
# Fallback models when the selected model isn't accessible (e.g., org not verified for GPT-5)
MODEL_FALLBACK_ORDER = ["gpt-4o", "gpt-4o-mini"]
def _log_error(msg: str):
"""Log error to stderr."""
sys.stderr.write(f"[REDDIT ERROR] {msg}\n")
sys.stderr.flush()
def _log_info(msg: str):
"""Log info to stderr."""
sys.stderr.write(f"[REDDIT] {msg}\n")
sys.stderr.flush()
def _is_model_access_error(error: http.HTTPError) -> bool:
"""Check if error is due to model access/verification issues."""
if error.status_code != 400:
return False
if not error.body:
return False
body_lower = error.body.lower()
# Check for common access/verification error messages
return any(phrase in body_lower for phrase in [
"verified",
"organization must be",
"does not have access",
"not available",
"not found",
])
OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
# Depth configurations: (min, max) threads to request
@@ -113,29 +140,50 @@ def search_reddit(
# Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
# Build list of models to try: requested model first, then fallbacks
models_to_try = [model] + [m for m in MODEL_FALLBACK_ORDER if m != model]
# Note: allowed_domains accepts base domain, not subdomains
# We rely on prompt to filter out developers.reddit.com, etc.
payload = {
"model": model,
"tools": [
{
"type": "web_search",
"filters": {
"allowed_domains": ["reddit.com"]
}
}
],
"include": ["web_search_call.action.sources"],
"input": REDDIT_SEARCH_PROMPT.format(
topic=topic,
from_date=from_date,
to_date=to_date,
min_items=min_items,
max_items=max_items,
),
}
input_text = REDDIT_SEARCH_PROMPT.format(
topic=topic,
from_date=from_date,
to_date=to_date,
min_items=min_items,
max_items=max_items,
)
return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
last_error = None
for current_model in models_to_try:
payload = {
"model": current_model,
"tools": [
{
"type": "web_search",
"filters": {
"allowed_domains": ["reddit.com"]
}
}
],
"include": ["web_search_call.action.sources"],
"input": input_text,
}
try:
return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
except http.HTTPError as e:
last_error = e
if _is_model_access_error(e):
_log_info(f"Model {current_model} not accessible, trying fallback...")
continue
# Non-access error, don't retry with different model
raise
# All models failed with access errors
if last_error:
_log_error(f"All models failed. Last error: {last_error}")
raise last_error
raise http.HTTPError("No models available")
def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+77
View File
@@ -0,0 +1,77 @@
"""Tests for openai_reddit module."""
import sys
import unittest
from pathlib import Path
# Add scripts directory to path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
from lib import http
from lib.openai_reddit import _is_model_access_error, MODEL_FALLBACK_ORDER
class TestIsModelAccessError(unittest.TestCase):
"""Tests for _is_model_access_error function."""
def test_returns_false_for_non_400_error(self):
"""Non-400 errors should not trigger fallback."""
error = http.HTTPError("Server error", status_code=500, body="Internal error")
self.assertFalse(_is_model_access_error(error))
def test_returns_false_for_400_without_body(self):
"""400 without body should not trigger fallback."""
error = http.HTTPError("Bad request", status_code=400, body=None)
self.assertFalse(_is_model_access_error(error))
def test_returns_true_for_verification_error(self):
"""Verification error should trigger fallback."""
error = http.HTTPError(
"Bad request",
status_code=400,
body='{"error": {"message": "Your organization must be verified to use the model \'gpt-5.2\'"}}'
)
self.assertTrue(_is_model_access_error(error))
def test_returns_true_for_access_error(self):
"""Access denied error should trigger fallback."""
error = http.HTTPError(
"Bad request",
status_code=400,
body='{"error": {"message": "Your account does not have access to this model"}}'
)
self.assertTrue(_is_model_access_error(error))
def test_returns_true_for_model_not_found(self):
"""Model not found error should trigger fallback."""
error = http.HTTPError(
"Bad request",
status_code=400,
body='{"error": {"message": "The model gpt-5.2 was not found"}}'
)
self.assertTrue(_is_model_access_error(error))
def test_returns_false_for_unrelated_400(self):
"""Unrelated 400 errors should not trigger fallback."""
error = http.HTTPError(
"Bad request",
status_code=400,
body='{"error": {"message": "Invalid JSON in request body"}}'
)
self.assertFalse(_is_model_access_error(error))
class TestModelFallbackOrder(unittest.TestCase):
"""Tests for MODEL_FALLBACK_ORDER constant."""
def test_contains_gpt4o(self):
"""Fallback list should include gpt-4o."""
self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER)
def test_gpt4o_is_first(self):
"""gpt-4o should be the first fallback option."""
self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4o")
if __name__ == "__main__":
unittest.main()