fix: YAML frontmatter and GPT-5 model fallback
- Fix invalid YAML in SKILL.md argument-hint (closes #8) Wrapped value in single quotes to properly escape double quotes - Add automatic model fallback for GPT-5 access errors (closes #9) When OpenAI returns 400 for unverified orgs, retry with gpt-4o - Add tests for model fallback logic Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
name: last30days
|
name: last30days
|
||||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
||||||
context: fork
|
context: fork
|
||||||
agent: Explore
|
agent: Explore
|
||||||
disable-model-invocation: true
|
disable-model-invocation: true
|
||||||
|
|||||||
@@ -7,12 +7,39 @@ from typing import Any, Dict, List, Optional
|
|||||||
|
|
||||||
from . import http
|
from . import http
|
||||||
|
|
||||||
|
# Fallback models when the selected model isn't accessible (e.g., org not verified for GPT-5)
|
||||||
|
MODEL_FALLBACK_ORDER = ["gpt-4o", "gpt-4o-mini"]
|
||||||
|
|
||||||
|
|
||||||
def _log_error(msg: str):
|
def _log_error(msg: str):
|
||||||
"""Log error to stderr."""
|
"""Log error to stderr."""
|
||||||
sys.stderr.write(f"[REDDIT ERROR] {msg}\n")
|
sys.stderr.write(f"[REDDIT ERROR] {msg}\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _log_info(msg: str):
|
||||||
|
"""Log info to stderr."""
|
||||||
|
sys.stderr.write(f"[REDDIT] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_model_access_error(error: http.HTTPError) -> bool:
|
||||||
|
"""Check if error is due to model access/verification issues."""
|
||||||
|
if error.status_code != 400:
|
||||||
|
return False
|
||||||
|
if not error.body:
|
||||||
|
return False
|
||||||
|
body_lower = error.body.lower()
|
||||||
|
# Check for common access/verification error messages
|
||||||
|
return any(phrase in body_lower for phrase in [
|
||||||
|
"verified",
|
||||||
|
"organization must be",
|
||||||
|
"does not have access",
|
||||||
|
"not available",
|
||||||
|
"not found",
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
||||||
|
|
||||||
# Depth configurations: (min, max) threads to request
|
# Depth configurations: (min, max) threads to request
|
||||||
@@ -113,29 +140,50 @@ def search_reddit(
|
|||||||
# Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
|
# Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
|
||||||
timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
|
timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
|
||||||
|
|
||||||
|
# Build list of models to try: requested model first, then fallbacks
|
||||||
|
models_to_try = [model] + [m for m in MODEL_FALLBACK_ORDER if m != model]
|
||||||
|
|
||||||
# Note: allowed_domains accepts base domain, not subdomains
|
# Note: allowed_domains accepts base domain, not subdomains
|
||||||
# We rely on prompt to filter out developers.reddit.com, etc.
|
# We rely on prompt to filter out developers.reddit.com, etc.
|
||||||
payload = {
|
input_text = REDDIT_SEARCH_PROMPT.format(
|
||||||
"model": model,
|
topic=topic,
|
||||||
"tools": [
|
from_date=from_date,
|
||||||
{
|
to_date=to_date,
|
||||||
"type": "web_search",
|
min_items=min_items,
|
||||||
"filters": {
|
max_items=max_items,
|
||||||
"allowed_domains": ["reddit.com"]
|
)
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"include": ["web_search_call.action.sources"],
|
|
||||||
"input": REDDIT_SEARCH_PROMPT.format(
|
|
||||||
topic=topic,
|
|
||||||
from_date=from_date,
|
|
||||||
to_date=to_date,
|
|
||||||
min_items=min_items,
|
|
||||||
max_items=max_items,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
|
last_error = None
|
||||||
|
for current_model in models_to_try:
|
||||||
|
payload = {
|
||||||
|
"model": current_model,
|
||||||
|
"tools": [
|
||||||
|
{
|
||||||
|
"type": "web_search",
|
||||||
|
"filters": {
|
||||||
|
"allowed_domains": ["reddit.com"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"include": ["web_search_call.action.sources"],
|
||||||
|
"input": input_text,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
|
||||||
|
except http.HTTPError as e:
|
||||||
|
last_error = e
|
||||||
|
if _is_model_access_error(e):
|
||||||
|
_log_info(f"Model {current_model} not accessible, trying fallback...")
|
||||||
|
continue
|
||||||
|
# Non-access error, don't retry with different model
|
||||||
|
raise
|
||||||
|
|
||||||
|
# All models failed with access errors
|
||||||
|
if last_error:
|
||||||
|
_log_error(f"All models failed. Last error: {last_error}")
|
||||||
|
raise last_error
|
||||||
|
raise http.HTTPError("No models available")
|
||||||
|
|
||||||
|
|
||||||
def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
"""Tests for openai_reddit module."""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Add scripts directory to path
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||||
|
|
||||||
|
from lib import http
|
||||||
|
from lib.openai_reddit import _is_model_access_error, MODEL_FALLBACK_ORDER
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsModelAccessError(unittest.TestCase):
|
||||||
|
"""Tests for _is_model_access_error function."""
|
||||||
|
|
||||||
|
def test_returns_false_for_non_400_error(self):
|
||||||
|
"""Non-400 errors should not trigger fallback."""
|
||||||
|
error = http.HTTPError("Server error", status_code=500, body="Internal error")
|
||||||
|
self.assertFalse(_is_model_access_error(error))
|
||||||
|
|
||||||
|
def test_returns_false_for_400_without_body(self):
|
||||||
|
"""400 without body should not trigger fallback."""
|
||||||
|
error = http.HTTPError("Bad request", status_code=400, body=None)
|
||||||
|
self.assertFalse(_is_model_access_error(error))
|
||||||
|
|
||||||
|
def test_returns_true_for_verification_error(self):
|
||||||
|
"""Verification error should trigger fallback."""
|
||||||
|
error = http.HTTPError(
|
||||||
|
"Bad request",
|
||||||
|
status_code=400,
|
||||||
|
body='{"error": {"message": "Your organization must be verified to use the model \'gpt-5.2\'"}}'
|
||||||
|
)
|
||||||
|
self.assertTrue(_is_model_access_error(error))
|
||||||
|
|
||||||
|
def test_returns_true_for_access_error(self):
|
||||||
|
"""Access denied error should trigger fallback."""
|
||||||
|
error = http.HTTPError(
|
||||||
|
"Bad request",
|
||||||
|
status_code=400,
|
||||||
|
body='{"error": {"message": "Your account does not have access to this model"}}'
|
||||||
|
)
|
||||||
|
self.assertTrue(_is_model_access_error(error))
|
||||||
|
|
||||||
|
def test_returns_true_for_model_not_found(self):
|
||||||
|
"""Model not found error should trigger fallback."""
|
||||||
|
error = http.HTTPError(
|
||||||
|
"Bad request",
|
||||||
|
status_code=400,
|
||||||
|
body='{"error": {"message": "The model gpt-5.2 was not found"}}'
|
||||||
|
)
|
||||||
|
self.assertTrue(_is_model_access_error(error))
|
||||||
|
|
||||||
|
def test_returns_false_for_unrelated_400(self):
|
||||||
|
"""Unrelated 400 errors should not trigger fallback."""
|
||||||
|
error = http.HTTPError(
|
||||||
|
"Bad request",
|
||||||
|
status_code=400,
|
||||||
|
body='{"error": {"message": "Invalid JSON in request body"}}'
|
||||||
|
)
|
||||||
|
self.assertFalse(_is_model_access_error(error))
|
||||||
|
|
||||||
|
|
||||||
|
class TestModelFallbackOrder(unittest.TestCase):
|
||||||
|
"""Tests for MODEL_FALLBACK_ORDER constant."""
|
||||||
|
|
||||||
|
def test_contains_gpt4o(self):
|
||||||
|
"""Fallback list should include gpt-4o."""
|
||||||
|
self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER)
|
||||||
|
|
||||||
|
def test_gpt4o_is_first(self):
|
||||||
|
"""gpt-4o should be the first fallback option."""
|
||||||
|
self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4o")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user