Initial import of NousResearch/hermes-agent

plugins/video_gen/fal/__init__.py

@@ -0,0 +1,613 @@
+"""FAL.ai video generation backend.
+
+User-facing surface: pick a **model family** (e.g. "Pixverse v6",
+"Veo 3.1", "Seedance 2.0", "Kling v3 4K", "LTX 2.3", "Happy Horse").
+The plugin auto-routes to the family's text-to-video endpoint when
+called without ``image_url``, and to its image-to-video endpoint when
+``image_url`` is provided. The agent never sees the routing — it just
+calls ``video_generate(prompt=..., image_url=...)``.
+
+Model families (each with t2v + i2v endpoints):
+
+  Cheap tier:
+    ltx-2.3       fal-ai/ltx-2.3-22b/text-to-video               /  fal-ai/ltx-2.3-22b/image-to-video
+    pixverse-v6   fal-ai/pixverse/v6/text-to-video               /  fal-ai/pixverse/v6/image-to-video
+
+  Premium tier:
+    veo3.1        fal-ai/veo3.1                                  /  fal-ai/veo3.1/image-to-video
+    seedance-2.0  bytedance/seedance-2.0/text-to-video           /  bytedance/seedance-2.0/image-to-video
+    kling-v3-4k   fal-ai/kling-video/v3/4k/text-to-video         /  fal-ai/kling-video/v3/4k/image-to-video
+    happy-horse   alibaba/happy-horse/text-to-video              /  alibaba/happy-horse/image-to-video
+
+Selection precedence for the active family:
+    1. ``model=`` arg from the tool call
+    2. ``FAL_VIDEO_MODEL`` env var
+    3. ``video_gen.fal.model`` in ``config.yaml``
+    4. ``video_gen.model`` in ``config.yaml`` (when it's one of our family IDs)
+    5. ``DEFAULT_MODEL``
+
+Authentication via ``FAL_KEY`` or the managed Nous gateway. Output is an
+HTTPS URL from FAL's CDN; the gateway downloads and delivers it.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+from agent.video_gen_provider import (
+    VideoGenProvider,
+    error_response,
+    success_response,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Family catalog
+# ---------------------------------------------------------------------------
+#
+# Each family declares both endpoints (when available) plus a per-family
+# capability sheet derived from FAL's OpenAPI schemas. Capability flags
+# drive which keys get added to the request payload — keys a family doesn't
+# advertise are dropped before send.
+#
+# Capabilities:
+#   aspect_ratios  : tuple of supported ratios (None = endpoint decides)
+#   resolutions    : tuple of supported resolutions (None = endpoint decides)
+#   durations      : tuple of supported durations OR (min, max) range
+#                    (heuristic: 2-element with gap > 1 is a range)
+#   audio          : True if generate_audio is supported
+#   negative       : True if negative_prompt is supported
+
+FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
+    # ─── Cheap / fast tier ─────────────────────────────────────────────
+    "ltx-2.3": {
+        "display": "LTX 2.3 (22B)",
+        "speed": "~30-60s",
+        "price": "cheap",
+        "strengths": "22B model with native audio generation. Affordable.",
+        "tier": "cheap",
+        "text_endpoint": "fal-ai/ltx-2.3-22b/text-to-video",
+        "image_endpoint": "fal-ai/ltx-2.3-22b/image-to-video",
+        # LTX docs don't expose duration/aspect/resolution enums — leave
+        # blank so we don't send unrecognized payload keys.
+        "aspect_ratios": None,
+        "resolutions": None,
+        "durations": None,
+        "audio": True,
+        "negative": True,
+    },
+    "pixverse-v6": {
+        "display": "Pixverse v6",
+        "speed": "~30-90s",
+        "price": "cheap",
+        "strengths": "Affordable. Negative prompts. 1-15s durations.",
+        "tier": "cheap",
+        "text_endpoint": "fal-ai/pixverse/v6/text-to-video",
+        "image_endpoint": "fal-ai/pixverse/v6/image-to-video",
+        "aspect_ratios": None,
+        "resolutions": ("360p", "540p", "720p", "1080p"),
+        "durations": (1, 15),
+        "audio": True,
+        "negative": True,
+    },
+    # ─── Expensive / premium tier ──────────────────────────────────────
+    "veo3.1": {
+        "display": "Veo 3.1",
+        "speed": "~60-120s",
+        "price": "premium",
+        "strengths": "Google DeepMind. Cinematic, native audio, strong prompt adherence.",
+        "tier": "premium",
+        "text_endpoint": "fal-ai/veo3.1",
+        "image_endpoint": "fal-ai/veo3.1/image-to-video",
+        "aspect_ratios": ("16:9", "9:16"),
+        "resolutions": ("720p", "1080p", "4k"),
+        "durations": (4, 6, 8),
+        "duration_suffix": "s",  # FAL veo3.1 wants "4s" not "4"
+        "audio": True,
+        "negative": True,
+    },
+    "seedance-2.0": {
+        "display": "Seedance 2.0",
+        "speed": "~60-120s",
+        "price": "premium",
+        "strengths": "ByteDance. Cinematic, synchronized audio + lip-sync, 4-15s.",
+        "tier": "premium",
+        "text_endpoint": "bytedance/seedance-2.0/text-to-video",
+        "image_endpoint": "bytedance/seedance-2.0/image-to-video",
+        # Seedance accepts "auto" too — we omit it from the enum so the
+        # agent can't pass it; the endpoint defaults handle the rest.
+        "aspect_ratios": ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16"),
+        "resolutions": ("480p", "720p", "1080p"),
+        "durations": (4, 15),
+        "audio": True,
+        "negative": False,
+    },
+    "kling-v3-4k": {
+        "display": "Kling v3 4K",
+        "speed": "~120-300s",
+        "price": "premium",
+        "strengths": "4K output, native audio (Chinese/English), 3-15s.",
+        "tier": "premium",
+        "text_endpoint": "fal-ai/kling-video/v3/4k/text-to-video",
+        "image_endpoint": "fal-ai/kling-video/v3/4k/image-to-video",
+        # Kling 4K image-to-video uses `start_image_url` instead of
+        # `image_url`. Handled in _build_payload via image_param_key.
+        "image_param_key": "start_image_url",
+        "aspect_ratios": ("16:9", "9:16", "1:1"),
+        "resolutions": None,  # 4K is implicit
+        "durations": (3, 15),
+        "audio": True,
+        "negative": True,
+    },
+    "happy-horse": {
+        "display": "Happy Horse 1.0",
+        "speed": "~60-120s",
+        "price": "premium",
+        "strengths": "Alibaba. New model, sparse public docs — conservative defaults.",
+        "tier": "premium",
+        "text_endpoint": "alibaba/happy-horse/text-to-video",
+        "image_endpoint": "alibaba/happy-horse/image-to-video",
+        # Docs don't expose duration/aspect/resolution — let the endpoint
+        # apply its own defaults.
+        "aspect_ratios": None,
+        "resolutions": None,
+        "durations": None,
+        "audio": False,
+        "negative": False,
+    },
+}
+
+DEFAULT_MODEL = "pixverse-v6"  # cheap, both modalities, sane defaults
+
+
+def _is_duration_range(durations: Any) -> bool:
+    """Heuristic: a 2-tuple of ints with a gap > 1 is treated as ``(min, max)``."""
+    if not isinstance(durations, tuple) or len(durations) != 2:
+        return False
+    if not all(isinstance(d, int) for d in durations):
+        return False
+    return durations[1] - durations[0] > 1
+
+
+def _clamp_duration(family: Dict[str, Any], duration: Optional[int]) -> Optional[int]:
+    durations = family.get("durations")
+    if not durations:
+        return duration
+    if duration is None:
+        return durations[0]
+    if _is_duration_range(durations):
+        lo, hi = durations
+        return max(lo, min(hi, duration))
+    # enum
+    if duration in durations:
+        return duration
+    return min(durations, key=lambda d: abs(d - duration))
+
+
+# ---------------------------------------------------------------------------
+# Config / model resolution
+# ---------------------------------------------------------------------------
+
+
+def _load_video_gen_section() -> Dict[str, Any]:
+    try:
+        from hermes_cli.config import load_config
+
+        cfg = load_config()
+        section = cfg.get("video_gen") if isinstance(cfg, dict) else None
+        return section if isinstance(section, dict) else {}
+    except Exception as exc:
+        logger.debug("Could not load video_gen config: %s", exc)
+        return {}
+
+
+def _resolve_family(explicit: Optional[str]) -> Tuple[str, Dict[str, Any]]:
+    """Decide which FAL family to use. Returns ``(family_id, meta)``."""
+    candidates: List[Optional[str]] = []
+    candidates.append(explicit)
+    candidates.append(os.environ.get("FAL_VIDEO_MODEL"))
+
+    cfg = _load_video_gen_section()
+    fal_cfg = cfg.get("fal") if isinstance(cfg.get("fal"), dict) else {}
+    if isinstance(fal_cfg, dict):
+        candidates.append(fal_cfg.get("model"))
+    top = cfg.get("model")
+    if isinstance(top, str):
+        candidates.append(top)
+
+    for c in candidates:
+        if isinstance(c, str) and c.strip() and c.strip() in FAL_FAMILIES:
+            fid = c.strip()
+            return fid, FAL_FAMILIES[fid]
+
+    return DEFAULT_MODEL, FAL_FAMILIES[DEFAULT_MODEL]
+
+
+# ---------------------------------------------------------------------------
+# Payload construction
+# ---------------------------------------------------------------------------
+
+
+def _build_payload(
+    family: Dict[str, Any],
+    *,
+    prompt: str,
+    image_url: Optional[str],
+    duration: Optional[int],
+    aspect_ratio: str,
+    resolution: str,
+    negative_prompt: Optional[str],
+    audio: Optional[bool],
+    seed: Optional[int],
+) -> Dict[str, Any]:
+    """Build a family-specific payload, dropping keys the family doesn't declare."""
+    payload: Dict[str, Any] = {}
+
+    if prompt:
+        payload["prompt"] = prompt
+    if image_url:
+        # Some endpoints (e.g. Kling v3 4K image-to-video) expect
+        # `start_image_url` instead of `image_url`. The family entry can
+        # declare an override.
+        key = family.get("image_param_key") or "image_url"
+        payload[key] = image_url
+    if seed is not None:
+        payload["seed"] = seed
+
+    if family.get("aspect_ratios"):
+        if aspect_ratio in family["aspect_ratios"]:
+            payload["aspect_ratio"] = aspect_ratio
+        # otherwise let the endpoint auto-crop / use its default
+
+    if family.get("resolutions"):
+        if resolution in family["resolutions"]:
+            payload["resolution"] = resolution
+        # else: let the endpoint default
+
+    clamped = _clamp_duration(family, duration)
+    if clamped is not None and family.get("durations"):
+        # FAL exposes duration as a string in the queue API ("8" not 8).
+        # Some families (e.g. veo3.1) require a unit suffix ("4s" not "4").
+        suffix = family.get("duration_suffix", "")
+        payload["duration"] = f"{clamped}{suffix}"
+
+    if family.get("audio") and audio is not None:
+        payload["generate_audio"] = bool(audio)
+
+    if family.get("negative") and negative_prompt:
+        payload["negative_prompt"] = negative_prompt
+
+    return payload
+
+
+# ---------------------------------------------------------------------------
+# fal_client lazy import (shared with image_generation_tool via fal_common)
+# ---------------------------------------------------------------------------
+
+_fal_client: Any = None
+
+
+def _load_fal_client() -> Any:
+    """Lazy-load the ``fal_client`` SDK and cache it on this module.
+
+    Delegates the actual import to :func:`tools.fal_common.import_fal_client`
+    so the ``lazy_deps`` ensure-install handling stays in one place.
+    """
+    global _fal_client
+    if _fal_client is not None:
+        return _fal_client
+    from tools.fal_common import import_fal_client
+    _fal_client = import_fal_client()
+    return _fal_client
+
+
+# ---------------------------------------------------------------------------
+# Managed FAL gateway (Nous Subscription)
+# ---------------------------------------------------------------------------
+
+_managed_fal_video_client: Any = None
+_managed_fal_video_client_config: Any = None
+_managed_fal_video_client_lock = threading.Lock()
+
+
+def _resolve_managed_fal_video_gateway():
+    """Return managed fal-queue gateway config when the user prefers the gateway
+    or direct FAL credentials are absent."""
+    from tools.tool_backend_helpers import fal_key_is_configured, prefers_gateway
+
+    if fal_key_is_configured() and not prefers_gateway("video_gen"):
+        return None
+    from tools.managed_tool_gateway import resolve_managed_tool_gateway
+
+    return resolve_managed_tool_gateway("fal-queue")
+
+
+def _get_managed_fal_video_client(managed_gateway):
+    """Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
+    global _managed_fal_video_client, _managed_fal_video_client_config
+    from tools.fal_common import _ManagedFalSyncClient
+
+    client_config = (
+        managed_gateway.gateway_origin.rstrip("/"),
+        managed_gateway.nous_user_token,
+    )
+    with _managed_fal_video_client_lock:
+        if _managed_fal_video_client is not None and _managed_fal_video_client_config == client_config:
+            return _managed_fal_video_client
+
+        _load_fal_client()
+        _managed_fal_video_client = _ManagedFalSyncClient(
+            _fal_client,
+            key=managed_gateway.nous_user_token,
+            queue_run_origin=managed_gateway.gateway_origin,
+        )
+        _managed_fal_video_client_config = client_config
+        return _managed_fal_video_client
+
+
+def _submit_fal_video_request(endpoint: str, arguments: Dict[str, Any]):
+    """Submit a FAL video request using direct credentials or the managed queue gateway.
+
+    Returns a request handle whose ``.get()`` blocks until the result is ready.
+    """
+    _load_fal_client()
+    request_headers = {"x-idempotency-key": str(uuid.uuid4())}
+    managed_gateway = _resolve_managed_fal_video_gateway()
+    if managed_gateway is None:
+        return _fal_client.submit(endpoint, arguments=arguments, headers=request_headers)
+
+    managed_client = _get_managed_fal_video_client(managed_gateway)
+    try:
+        return managed_client.submit(
+            endpoint,
+            arguments=arguments,
+            headers=request_headers,
+        )
+    except Exception as exc:
+        from tools.fal_common import _extract_http_status
+
+        status = _extract_http_status(exc)
+        if status is not None and 400 <= status < 500:
+            raise ValueError(
+                f"Nous Subscription gateway rejected endpoint '{endpoint}' "
+                f"(HTTP {status}). This model may not yet be enabled on "
+                f"the Nous Portal's FAL proxy. Either:\n"
+                f"  • Set FAL_KEY in your environment to use FAL.ai directly, or\n"
+                f"  • Pick a different model via `hermes tools` → Video Generation."
+            ) from exc
+        raise
+
+
+def _check_fal_video_available() -> bool:
+    """True if the FAL.ai video backend is reachable (direct key or managed gateway)."""
+    from tools.tool_backend_helpers import fal_key_is_configured
+
+    if fal_key_is_configured():
+        return True
+    return _resolve_managed_fal_video_gateway() is not None
+
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+
+class FALVideoGenProvider(VideoGenProvider):
+    """FAL.ai multi-family video generation backend.
+
+    Routes between text-to-video and image-to-video endpoints automatically
+    based on whether ``image_url`` was provided.
+    """
+
+    @property
+    def name(self) -> str:
+        return "fal"
+
+    @property
+    def display_name(self) -> str:
+        return "FAL"
+
+    def is_available(self) -> bool:
+        try:
+            return _check_fal_video_available()
+        except Exception:  # noqa: BLE001 — never break the picker
+            return False
+
+    def list_models(self) -> List[Dict[str, Any]]:
+        out: List[Dict[str, Any]] = []
+        for fid, meta in FAL_FAMILIES.items():
+            modalities: List[str] = []
+            if meta.get("text_endpoint"):
+                modalities.append("text")
+            if meta.get("image_endpoint"):
+                modalities.append("image")
+            out.append({
+                "id": fid,
+                "display": meta["display"],
+                "speed": meta["speed"],
+                "strengths": meta["strengths"],
+                "price": meta["price"],
+                "tier": meta.get("tier", "premium"),
+                "modalities": modalities,
+            })
+        return out
+
+    def default_model(self) -> Optional[str]:
+        return DEFAULT_MODEL
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "FAL",
+            "badge": "paid",
+            "tag": "LTX, Pixverse, Veo 3.1, Seedance 2.0, Kling 4K, Happy Horse — text-to-video & image-to-video",
+            "env_vars": [
+                {
+                    "key": "FAL_KEY",
+                    "prompt": "FAL.ai API key",
+                    "url": "https://fal.ai/dashboard/keys",
+                },
+            ],
+        }
+
+    def capabilities(self) -> Dict[str, Any]:
+        return {
+            "modalities": ["text", "image"],
+            "aspect_ratios": ["16:9", "9:16", "1:1"],
+            "resolutions": ["360p", "540p", "720p", "1080p"],
+            "max_duration": 15,
+            "min_duration": 1,
+            "supports_audio": True,
+            "supports_negative_prompt": True,
+            "max_reference_images": 0,
+        }
+
+    def generate(
+        self,
+        prompt: str,
+        *,
+        model: Optional[str] = None,
+        image_url: Optional[str] = None,
+        reference_image_urls: Optional[List[str]] = None,
+        duration: Optional[int] = None,
+        aspect_ratio: str = "16:9",
+        resolution: str = "720p",
+        negative_prompt: Optional[str] = None,
+        audio: Optional[bool] = None,
+        seed: Optional[int] = None,
+        **kwargs: Any,
+    ) -> Dict[str, Any]:
+        if not _check_fal_video_available():
+            return error_response(
+                error=(
+                    "No FAL backend available. Either set FAL_KEY "
+                    "(run `hermes tools` → Video Generation → FAL to configure) "
+                    "or sign in to Nous (`hermes setup`) for managed gateway access."
+                ),
+                error_type="auth_required",
+                provider="fal",
+                prompt=prompt,
+            )
+
+        try:
+            _load_fal_client()
+        except ImportError:
+            return error_response(
+                error="fal_client Python package not installed (pip install fal-client)",
+                error_type="missing_dependency",
+                provider="fal",
+                prompt=prompt,
+            )
+
+        prompt = (prompt or "").strip()
+        family_id, family = _resolve_family(model)
+
+        # Route: image_url → image-to-video endpoint; else → text-to-video.
+        image_url_norm = (image_url or "").strip() or None
+        if image_url_norm:
+            endpoint = family.get("image_endpoint")
+            modality_used = "image"
+            if not endpoint:
+                return error_response(
+                    error=(
+                        f"FAL family {family_id} has no image-to-video "
+                        f"endpoint. Pick a family with image-to-video support "
+                        f"via `hermes tools` → Video Generation."
+                    ),
+                    error_type="modality_unsupported",
+                    provider="fal", model=family_id, prompt=prompt,
+                )
+        else:
+            endpoint = family.get("text_endpoint")
+            modality_used = "text"
+            if not endpoint:
+                return error_response(
+                    error=(
+                        f"FAL family {family_id} has no text-to-video "
+                        f"endpoint. Pass an image_url to use its "
+                        f"image-to-video endpoint, or pick a different family."
+                    ),
+                    error_type="modality_unsupported",
+                    provider="fal", model=family_id, prompt=prompt,
+                )
+
+        if not prompt:
+            return error_response(
+                error="prompt is required.",
+                error_type="missing_prompt",
+                provider="fal", model=family_id, prompt=prompt,
+            )
+
+        payload = _build_payload(
+            family,
+            prompt=prompt,
+            image_url=image_url_norm,
+            duration=duration,
+            aspect_ratio=aspect_ratio,
+            resolution=resolution,
+            negative_prompt=negative_prompt,
+            audio=audio,
+            seed=seed,
+        )
+
+        try:
+            handle = _submit_fal_video_request(endpoint, payload)
+            result = handle.get()
+        except Exception as exc:
+            logger.warning(
+                "FAL video gen failed (family=%s, endpoint=%s): %s",
+                family_id, endpoint, exc, exc_info=True,
+            )
+            return error_response(
+                error=f"FAL video generation failed: {exc}",
+                error_type="api_error",
+                provider="fal", model=family_id, prompt=prompt,
+                aspect_ratio=aspect_ratio,
+            )
+
+        video = (result or {}).get("video") if isinstance(result, dict) else None
+        url: Optional[str] = None
+        if isinstance(video, dict):
+            url = video.get("url")
+        elif isinstance(video, str):
+            url = video
+
+        if not url:
+            return error_response(
+                error="FAL returned no video URL in response",
+                error_type="empty_response",
+                provider="fal", model=family_id, prompt=prompt,
+            )
+
+        extra: Dict[str, Any] = {"endpoint": endpoint}
+        if isinstance(video, dict):
+            if video.get("file_size"):
+                extra["file_size"] = video["file_size"]
+            if video.get("content_type"):
+                extra["content_type"] = video["content_type"]
+
+        return success_response(
+            video=url,
+            model=family_id,
+            prompt=prompt,
+            modality=modality_used,
+            aspect_ratio=aspect_ratio if "aspect_ratio" in payload else "",
+            duration=int("".join(c for c in payload["duration"] if c.isdigit()) or "0") if "duration" in payload else 0,
+            provider="fal",
+            extra=extra,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Plugin entry point
+# ---------------------------------------------------------------------------
+
+
+def register(ctx) -> None:
+    """Plugin entry point — wire ``FALVideoGenProvider`` into the registry."""
+    ctx.register_video_gen_provider(FALVideoGenProvider())

plugins/video_gen/fal/plugin.yaml

@@ -0,0 +1,7 @@
+name: fal
+version: 1.0.0
+description: "FAL.ai video generation backend. Multi-model — Veo 3.1, Kling, Pixverse — covering text-to-video and image-to-video via fal_client's queue API."
+author: NousResearch
+kind: backend
+requires_env:
+  - FAL_KEY

plugins/video_gen/xai/__init__.py

@@ -0,0 +1,441 @@
+"""xAI Grok-Imagine video generation backend.
+
+Surface: text-to-video and image-to-video (animate an input image)
+through xAI's ``/videos/generations`` endpoint. Edit and extend are not
+exposed in this unified surface — xAI is the only backend that supports
+them and the inconsistency would force per-backend prose in the agent's
+tool description.
+
+Originally salvaged from PR #10600 by @Jaaneek; reshaped into the
+:class:`VideoGenProvider` plugin interface and trimmed to the
+generate-only surface.
+
+Authentication: xAI Grok OAuth tokens (preferred — billed against the
+user's SuperGrok or X Premium+ subscription) or ``XAI_API_KEY``. Both routes are
+resolved through ``tools.xai_http.resolve_xai_http_credentials`` so a
+single login covers chat + TTS + image gen + video gen + transcription.
+Output is an HTTPS URL from xAI's CDN; the gateway downloads and
+delivers it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+import httpx
+
+from agent.video_gen_provider import (
+    VideoGenProvider,
+    error_response,
+    success_response,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+DEFAULT_XAI_BASE_URL = "https://api.x.ai/v1"
+DEFAULT_MODEL = "grok-imagine-video"
+DEFAULT_DURATION = 8
+DEFAULT_ASPECT_RATIO = "16:9"
+DEFAULT_RESOLUTION = "720p"
+DEFAULT_TIMEOUT_SECONDS = 240
+DEFAULT_POLL_INTERVAL_SECONDS = 5
+
+VALID_ASPECT_RATIOS = {"1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"}
+VALID_RESOLUTIONS = {"480p", "720p"}
+MAX_REFERENCE_IMAGES = 7
+
+
+_MODELS: Dict[str, Dict[str, Any]] = {
+    "grok-imagine-video": {
+        "display": "Grok Imagine Video",
+        "speed": "~60-240s",
+        "strengths": "Text-to-video + image-to-video; up to 7 reference images for style/character.",
+        "price": "see https://docs.x.ai/docs/models",
+        "modalities": ["text", "image"],
+    },
+}
+
+
+# ---------------------------------------------------------------------------
+# HTTP helpers
+# ---------------------------------------------------------------------------
+
+
+def _resolve_xai_credentials() -> Tuple[str, str]:
+    """Return ``(api_key, base_url)`` from the shared xAI credential resolver.
+
+    Order: runtime provider (xai-oauth pool entry) → singleton ``auth.json``
+    OAuth tokens → ``XAI_API_KEY`` env var. ``api_key`` is empty when no
+    credential source is available; callers must check before using it.
+    """
+    try:
+        from tools.xai_http import resolve_xai_http_credentials
+
+        creds = resolve_xai_http_credentials() or {}
+    except Exception as exc:
+        logger.debug("xAI credential resolver failed: %s", exc)
+        creds = {}
+
+    api_key = str(creds.get("api_key") or os.getenv("XAI_API_KEY", "")).strip()
+    base_url = str(
+        creds.get("base_url")
+        or os.getenv("XAI_BASE_URL")
+        or DEFAULT_XAI_BASE_URL
+    ).strip().rstrip("/")
+    return api_key, base_url
+
+
+def _xai_user_agent() -> str:
+    try:
+        from tools.xai_http import hermes_xai_user_agent
+
+        return hermes_xai_user_agent()
+    except Exception:
+        return "hermes-agent/video_gen"
+
+
+def _xai_headers(api_key: str) -> Dict[str, str]:
+    return {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "User-Agent": _xai_user_agent(),
+    }
+
+
+def _normalize_reference_images(reference_image_urls: Optional[List[str]]):
+    refs = []
+    for url in reference_image_urls or []:
+        normalized = (url or "").strip()
+        if normalized:
+            refs.append({"url": normalized})
+    return refs or None
+
+
+def _clamp_duration(duration: Optional[int], has_reference_images: bool) -> int:
+    value = duration if duration is not None else DEFAULT_DURATION
+    if value < 1:
+        value = 1
+    if value > 15:
+        value = 15
+    if has_reference_images and value > 10:
+        value = 10
+    return value
+
+
+async def _submit(
+    client: httpx.AsyncClient,
+    payload: Dict[str, Any],
+    *,
+    api_key: str,
+    base_url: str,
+) -> str:
+    """POST to /videos/generations — xAI's only public endpoint for our
+    text-to-video and image-to-video surface."""
+    response = await client.post(
+        f"{base_url}/videos/generations",
+        headers={**_xai_headers(api_key), "x-idempotency-key": str(uuid.uuid4())},
+        json=payload,
+        timeout=60,
+    )
+    response.raise_for_status()
+    body = response.json()
+    request_id = body.get("request_id")
+    if not request_id:
+        raise RuntimeError("xAI video response did not include request_id")
+    return request_id
+
+
+async def _poll(
+    client: httpx.AsyncClient,
+    request_id: str,
+    *,
+    api_key: str,
+    base_url: str,
+    timeout_seconds: int,
+    poll_interval: int,
+) -> Dict[str, Any]:
+    elapsed = 0.0
+    last_status = "queued"
+    while elapsed < timeout_seconds:
+        response = await client.get(
+            f"{base_url}/videos/{request_id}",
+            headers=_xai_headers(api_key),
+            timeout=30,
+        )
+        response.raise_for_status()
+        body = response.json()
+        last_status = (body.get("status") or "").lower()
+
+        if last_status == "done":
+            return {"status": "done", "body": body}
+        if last_status in {"failed", "error", "expired", "cancelled"}:
+            return {"status": last_status, "body": body}
+
+        await asyncio.sleep(poll_interval)
+        elapsed += poll_interval
+
+    return {"status": "timeout", "body": {"status": last_status}}
+
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+
+class XAIVideoGenProvider(VideoGenProvider):
+    """xAI grok-imagine-video backend (text-to-video + image-to-video)."""
+
+    @property
+    def name(self) -> str:
+        return "xai"
+
+    @property
+    def display_name(self) -> str:
+        return "xAI"
+
+    def is_available(self) -> bool:
+        api_key, _ = _resolve_xai_credentials()
+        return bool(api_key)
+
+    def list_models(self) -> List[Dict[str, Any]]:
+        return [{"id": mid, **meta} for mid, meta in _MODELS.items()]
+
+    def default_model(self) -> Optional[str]:
+        return DEFAULT_MODEL
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        # Auth resolution lives entirely in the shared ``xai_grok`` post_setup
+        # hook (``hermes_cli/tools_config.py``) so the picker doesn't blindly
+        # prompt for an API key when the user is already signed in via xAI
+        # Grok OAuth (SuperGrok / Premium+) — TTS / image gen / video gen
+        # all share the same credential resolver. The hook offers an
+        # OAuth-vs-API-key choice when neither is configured.
+        return {
+            "name": "xAI Grok Imagine",
+            "badge": "paid",
+            "tag": "grok-imagine-video — text-to-video & image-to-video; uses xAI Grok OAuth or XAI_API_KEY",
+            "env_vars": [],
+            "post_setup": "xai_grok",
+        }
+
+    def capabilities(self) -> Dict[str, Any]:
+        return {
+            "modalities": ["text", "image"],
+            "aspect_ratios": sorted(VALID_ASPECT_RATIOS),
+            "resolutions": sorted(VALID_RESOLUTIONS),
+            "max_duration": 15,
+            "min_duration": 1,
+            "supports_audio": False,
+            "supports_negative_prompt": False,
+            "max_reference_images": MAX_REFERENCE_IMAGES,
+        }
+
+    def generate(
+        self,
+        prompt: str,
+        *,
+        model: Optional[str] = None,
+        image_url: Optional[str] = None,
+        reference_image_urls: Optional[List[str]] = None,
+        duration: Optional[int] = None,
+        aspect_ratio: str = DEFAULT_ASPECT_RATIO,
+        resolution: str = DEFAULT_RESOLUTION,
+        negative_prompt: Optional[str] = None,
+        audio: Optional[bool] = None,
+        seed: Optional[int] = None,
+        **kwargs: Any,
+    ) -> Dict[str, Any]:
+        try:
+            loop = asyncio.new_event_loop()
+            try:
+                return loop.run_until_complete(self._generate_async(
+                    prompt=prompt,
+                    model=model,
+                    image_url=image_url,
+                    reference_image_urls=reference_image_urls,
+                    duration=duration,
+                    aspect_ratio=aspect_ratio,
+                    resolution=resolution,
+                ))
+            finally:
+                loop.close()
+        except Exception as exc:
+            logger.warning("xAI video gen unexpected failure: %s", exc, exc_info=True)
+            return error_response(
+                error=f"xAI video generation failed: {exc}",
+                error_type="api_error",
+                provider="xai",
+                model=model or DEFAULT_MODEL,
+                prompt=prompt,
+                aspect_ratio=aspect_ratio,
+            )
+
+    async def _generate_async(
+        self,
+        *,
+        prompt: str,
+        model: Optional[str],
+        image_url: Optional[str],
+        reference_image_urls: Optional[List[str]],
+        duration: Optional[int],
+        aspect_ratio: str,
+        resolution: str,
+    ) -> Dict[str, Any]:
+        api_key, base_url = _resolve_xai_credentials()
+        if not api_key:
+            return error_response(
+                error=(
+                    "No xAI credentials found. Sign in via `hermes auth add xai-oauth` "
+                    "(SuperGrok / Premium+) or set XAI_API_KEY from "
+                    "https://console.x.ai/."
+                ),
+                error_type="auth_required",
+                provider="xai", prompt=prompt,
+            )
+
+        prompt = (prompt or "").strip()
+        image_url_norm = (image_url or "").strip() or None
+        normalized_aspect_ratio = (aspect_ratio or DEFAULT_ASPECT_RATIO).strip()
+        normalized_resolution = (resolution or DEFAULT_RESOLUTION).strip().lower()
+        modality_used = "image" if image_url_norm else "text"
+
+        if not prompt:
+            return error_response(
+                error=(
+                    "prompt is required for xAI video generation "
+                    "(text-to-video or image-to-video)"
+                ),
+                error_type="missing_prompt",
+                provider="xai", prompt=prompt,
+            )
+
+        refs = _normalize_reference_images(reference_image_urls)
+        if refs and len(refs) > MAX_REFERENCE_IMAGES:
+            return error_response(
+                error=f"reference_image_urls supports at most {MAX_REFERENCE_IMAGES} images on xAI",
+                error_type="too_many_references",
+                provider="xai", prompt=prompt,
+            )
+        if image_url_norm and refs:
+            return error_response(
+                error="image_url and reference_image_urls cannot be combined on xAI",
+                error_type="conflicting_inputs",
+                provider="xai", prompt=prompt,
+            )
+
+        clamped_duration = _clamp_duration(duration, has_reference_images=bool(refs))
+
+        if normalized_aspect_ratio not in VALID_ASPECT_RATIOS:
+            normalized_aspect_ratio = DEFAULT_ASPECT_RATIO
+        if normalized_resolution not in VALID_RESOLUTIONS:
+            normalized_resolution = DEFAULT_RESOLUTION
+
+        payload: Dict[str, Any] = {
+            "model": model or DEFAULT_MODEL,
+            "prompt": prompt,
+            "duration": clamped_duration,
+            "aspect_ratio": normalized_aspect_ratio,
+            "resolution": normalized_resolution,
+        }
+        if image_url_norm:
+            payload["image"] = {"url": image_url_norm}
+        if refs:
+            payload["reference_images"] = refs
+
+        async with httpx.AsyncClient() as client:
+            try:
+                request_id = await _submit(
+                    client, payload, api_key=api_key, base_url=base_url
+                )
+            except httpx.HTTPStatusError as exc:
+                detail = ""
+                try:
+                    detail = exc.response.text[:500]
+                except Exception:
+                    pass
+                return error_response(
+                    error=f"xAI submit failed ({exc.response.status_code}): {detail or exc}",
+                    error_type="api_error",
+                    provider="xai",
+                    model=model or DEFAULT_MODEL,
+                    prompt=prompt,
+                )
+
+            poll_result = await _poll(
+                client, request_id,
+                api_key=api_key, base_url=base_url,
+                timeout_seconds=DEFAULT_TIMEOUT_SECONDS,
+                poll_interval=DEFAULT_POLL_INTERVAL_SECONDS,
+            )
+
+        status = poll_result["status"]
+        body = poll_result["body"]
+
+        if status == "done":
+            video = body.get("video") or {}
+            url = video.get("url")
+            if not url:
+                return error_response(
+                    error="xAI video generation completed without a video URL",
+                    error_type="empty_response",
+                    provider="xai",
+                    model=body.get("model") or model or DEFAULT_MODEL,
+                    prompt=prompt,
+                )
+            extra: Dict[str, Any] = {
+                "request_id": request_id,
+                "resolution": normalized_resolution,
+            }
+            if body.get("usage"):
+                extra["usage"] = body["usage"]
+            return success_response(
+                video=url,
+                model=body.get("model") or model or DEFAULT_MODEL,
+                prompt=prompt,
+                modality=modality_used,
+                aspect_ratio=normalized_aspect_ratio,
+                duration=video.get("duration") or clamped_duration,
+                provider="xai",
+                extra=extra,
+            )
+
+        if status == "timeout":
+            return error_response(
+                error=f"Timed out waiting for video generation after {DEFAULT_TIMEOUT_SECONDS}s",
+                error_type="timeout",
+                provider="xai",
+                model=model or DEFAULT_MODEL,
+                prompt=prompt,
+            )
+
+        message = (
+            (body.get("error", {}) or {}).get("message")
+            or body.get("message")
+            or f"xAI video generation ended with status '{status}'"
+        )
+        return error_response(
+            error=message,
+            error_type=f"xai_{status}",
+            provider="xai",
+            model=model or DEFAULT_MODEL,
+            prompt=prompt,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Plugin entry point
+# ---------------------------------------------------------------------------
+
+
+def register(ctx) -> None:
+    """Plugin entry point — wire ``XAIVideoGenProvider`` into the registry."""
+    ctx.register_video_gen_provider(XAIVideoGenProvider())

plugins/video_gen/xai/plugin.yaml

@@ -0,0 +1,7 @@
+name: xai
+version: 1.0.0
+description: "xAI Grok-Imagine video generation backend. Supports text-to-video, image-to-video, reference-image-guided generation, video edit, and video extend via the xAI async videos API."
+author: NousResearch
+kind: backend
+requires_env:
+  - XAI_API_KEY