feat: add optional YouTube proxy support for IP blocks

Route transcript fetching through YOUTUBE_PROXY or Webshare residential
config, with clearer errors when YouTube still blocks requests.

Co-authored-by: Cursor <cursoragent@cursor.com>

.env.example

@@ -9,5 +9,10 @@ JWT_EXPIRE_DAYS=90
 OPENROUTER_API_KEY=
 OPENROUTER_MODEL=google/gemini-2.5-flash
 
-# Optional lokal
+# YouTube IP-Block umgehen (optional)
-# APP_URL=http://localhost:8000
+# Statischer Proxy aus dem Dashboard, z.B. http://USER:PASS@HOST:PORT
+YOUTUBE_PROXY=
+# Alternativ Webshare *Residential* (kostenpflichtig, rotating — NICHT Free/Proxy-Server):
+# WEBSHARE_PROXY_USERNAME=
+# WEBSHARE_PROXY_PASSWORD=
+# WEBSHARE_PROXY_LOCATIONS=de,es

README.md

@@ -64,6 +64,27 @@ Traefik übernimmt TLS via `myresolver`. Persistente Daten liegen in `./data/`.
 | `POST /api/recaps` | JWT | Recap erstellen |
 | `POST /api/recaps/{video_id}/regenerate` | JWT | Neu generieren |
 
+## YouTube IP-Block / Proxy
+
+YouTube blockiert oft Server-IPs (Hetzner, AWS, …). Optional in `.env`:
+
+```bash
+# Statischer Proxy (z.B. aus Webshare „Proxy Server“ / Dashboard-Zeile)
+YOUTUBE_PROXY=http://USER:PASS@HOST:PORT
+```
+
+Für yt-dlp-Fallback und VTT-Downloads wird dieselbe URL genutzt.
+
+**Webshare-Hinweis:** Der kostenlose Tier und „Proxy Server“ (einzelne IP wie im Dashboard) funktionieren *manchmal* für wenige Recaps pro Tag — laut `youtube-transcript-api` aber unzuverlässig. Rotating **Residential** (kostenpflichtig) ist die robuste Lösung:
+
+```bash
+WEBSHARE_PROXY_USERNAME=...
+WEBSHARE_PROXY_PASSWORD=...
+WEBSHARE_PROXY_LOCATIONS=de
+```
+
+Öffentliche Gratis-Proxy-Listen: nein (langsam, unsicher, meist schon gebannt).
+
 ## Stack
 
 - FastAPI + SQLite

compose.dev.yml

@@ -8,6 +8,10 @@ services:
       - JWT_SECRET=${JWT_SECRET:-dev-secret-change-me}
       - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-}
       - OPENROUTER_MODEL=${OPENROUTER_MODEL:-google/gemini-2.5-flash}
+      - YOUTUBE_PROXY=${YOUTUBE_PROXY:-}
+      - WEBSHARE_PROXY_USERNAME=${WEBSHARE_PROXY_USERNAME:-}
+      - WEBSHARE_PROXY_PASSWORD=${WEBSHARE_PROXY_PASSWORD:-}
+      - WEBSHARE_PROXY_LOCATIONS=${WEBSHARE_PROXY_LOCATIONS:-}
       - DB_PATH=/data/ytrecap.sqlite
       - APP_URL=http://localhost:8000
     volumes:

compose.yml

@@ -8,6 +8,10 @@ services:
       - JWT_EXPIRE_DAYS=${JWT_EXPIRE_DAYS:-90}
       - OPENROUTER_API_KEY=${OPENROUTER_API_KEY}
       - OPENROUTER_MODEL=${OPENROUTER_MODEL:-google/gemini-2.5-flash}
+      - YOUTUBE_PROXY=${YOUTUBE_PROXY:-}
+      - WEBSHARE_PROXY_USERNAME=${WEBSHARE_PROXY_USERNAME:-}
+      - WEBSHARE_PROXY_PASSWORD=${WEBSHARE_PROXY_PASSWORD:-}
+      - WEBSHARE_PROXY_LOCATIONS=${WEBSHARE_PROXY_LOCATIONS:-}
       - DB_PATH=/data/ytrecap.sqlite
       - APP_URL=https://ytrecap.schwenk.online
       - TZ=Europe/Berlin

server/settings.py

@@ -18,5 +18,13 @@ class Settings(BaseSettings):
     app_url: str = "http://localhost:8000"
     app_version: str = "1.0.0"
 
+    # YouTube transcript proxy (optional — for IP blocks)
+    # Single URL used for HTTP+HTTPS, e.g. http://user:pass@host:port
+    youtube_proxy: str = ""
+    # Webshare rotating residential (paid package — NOT free "Proxy Server" tier)
+    webshare_proxy_username: str = ""
+    webshare_proxy_password: str = ""
+    webshare_proxy_locations: str = ""  # comma-separated, e.g. de,es
+
 
 settings = Settings()  # type: ignore[call-arg]

server/transcripts.py

@@ -4,14 +4,20 @@ import hashlib
 import logging
 import re
 from dataclasses import dataclass
+from typing import Any
 
 from youtube_transcript_api import YouTubeTranscriptApi
 from youtube_transcript_api._errors import (
+    IpBlocked,
     NoTranscriptFound,
+    RequestBlocked,
     TranscriptsDisabled,
     VideoUnavailable,
     YouTubeRequestFailed,
 )
+from youtube_transcript_api.proxies import GenericProxyConfig, ProxyConfig, WebshareProxyConfig
+
+from server.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -59,14 +65,75 @@ def _normalize_ytt_segments(raw: list[dict]) -> list[TranscriptSegment]:
     return [TranscriptSegment(start=float(item["start"]), text=str(item["text"])) for item in raw]
 
 
+def _proxy_dict() -> dict[str, str] | None:
+    config = build_proxy_config()
+    if config is None:
+        return None
+    return config.to_requests_dict()
+
+
+def build_proxy_config() -> ProxyConfig | None:
+    if settings.webshare_proxy_username and settings.webshare_proxy_password:
+        locations = [
+            code.strip().lower()
+            for code in settings.webshare_proxy_locations.split(",")
+            if code.strip()
+        ]
+        return WebshareProxyConfig(
+            proxy_username=settings.webshare_proxy_username,
+            proxy_password=settings.webshare_proxy_password,
+            filter_ip_locations=locations or None,
+        )
+
+    proxy_url = settings.youtube_proxy.strip()
+    if proxy_url:
+        return GenericProxyConfig(http_url=proxy_url, https_url=proxy_url)
+
+    return None
+
+
+def proxy_url_for_ytdlp() -> str | None:
+    direct = settings.youtube_proxy.strip()
+    if direct:
+        return direct
+    config = build_proxy_config()
+    if config is None:
+        return None
+    proxies = config.to_requests_dict()
+    return proxies.get("https") or proxies.get("http")
+
+
+def build_youtube_transcript_api() -> YouTubeTranscriptApi:
+    config = build_proxy_config()
+    if config is None:
+        return YouTubeTranscriptApi()
+    return YouTubeTranscriptApi(proxy_config=config)
+
+
+def _blocked_message() -> str:
+    if build_proxy_config() is not None:
+        return (
+            "YouTube blockiert auch über den Proxy. "
+            "Statische/datacenter Proxies halten oft nicht lange — "
+            "für zuverlässigere Ergebnisse rotating residential nutzen."
+        )
+    return (
+        "YouTube blockiert deine Server-IP. "
+        "Setze YOUTUBE_PROXY in der .env (siehe README)."
+    )
+
+
 def _fetch_with_youtube_transcript_api(video_id: str) -> Transcript:
-    api = YouTubeTranscriptApi()
+    api = build_youtube_transcript_api()
     try:
         fetched = api.fetch(video_id, languages=list(PREFERRED_LANGUAGES))
         segments = _normalize_ytt_segments(
             [{"start": s.start, "text": s.text} for s in fetched]
         )
         return Transcript(language=fetched.language_code, segments=segments, source="youtube-transcript-api")
+    except (RequestBlocked, IpBlocked) as exc:
+        logger.warning("YouTube IP block for %s: %s", video_id, exc)
+        raise ValueError(_blocked_message()) from exc
     except (TranscriptsDisabled, NoTranscriptFound, VideoUnavailable):
         raise
     except YouTubeRequestFailed as exc:
@@ -104,7 +171,7 @@ def _fetch_with_ytdlp(video_id: str) -> Transcript:
     import yt_dlp
 
     url = f"https://www.youtube.com/watch?v={video_id}"
-    opts: dict = {
+    opts: dict[str, Any] = {
         "skip_download": True,
         "quiet": True,
         "no_warnings": True,
@@ -113,6 +180,10 @@ def _fetch_with_ytdlp(video_id: str) -> Transcript:
         "subtitleslangs": list(PREFERRED_LANGUAGES),
         "subtitlesformat": "vtt",
     }
+    proxy = proxy_url_for_ytdlp()
+    if proxy:
+        opts["proxy"] = proxy
+
     with yt_dlp.YoutubeDL(opts) as ydl:
         info = ydl.extract_info(url, download=False)
 
@@ -145,9 +216,17 @@ def _fetch_with_ytdlp(video_id: str) -> Transcript:
 
     import httpx
 
-    resp = httpx.get(vtt_url, timeout=30.0, follow_redirects=True)
+    client_kwargs: dict[str, Any] = {"timeout": 30.0, "follow_redirects": True}
-    resp.raise_for_status()
+    proxies = _proxy_dict()
-    segments = _parse_vtt(resp.text)
+    if proxies:
+        client_kwargs["proxy"] = proxies.get("https") or proxies.get("http")
+
+    with httpx.Client(**client_kwargs) as client:
+        resp = client.get(vtt_url)
+        resp.raise_for_status()
+        vtt_text = resp.text
+
+    segments = _parse_vtt(vtt_text)
     if not segments:
         raise ValueError("Untertitel konnten nicht geparst werden.")