From 76de7c7a3a150c7706457275bbf697ca654e1cc6 Mon Sep 17 00:00:00 2001 From: Frank Schwenk Date: Tue, 16 Jun 2026 16:23:06 +0200 Subject: [PATCH] feat: add optional YouTube proxy support for IP blocks Route transcript fetching through YOUTUBE_PROXY or Webshare residential config, with clearer errors when YouTube still blocks requests. Co-authored-by: Cursor --- .env.example | 9 ++++- README.md | 21 ++++++++++ compose.dev.yml | 4 ++ compose.yml | 4 ++ server/settings.py | 8 ++++ server/transcripts.py | 89 ++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 128 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 20dbd1d..fc65296 100644 --- a/.env.example +++ b/.env.example @@ -9,5 +9,10 @@ JWT_EXPIRE_DAYS=90 OPENROUTER_API_KEY= OPENROUTER_MODEL=google/gemini-2.5-flash -# Optional lokal -# APP_URL=http://localhost:8000 +# YouTube IP-Block umgehen (optional) +# Statischer Proxy aus dem Dashboard, z.B. http://USER:PASS@HOST:PORT +YOUTUBE_PROXY= +# Alternativ Webshare *Residential* (kostenpflichtig, rotating — NICHT Free/Proxy-Server): +# WEBSHARE_PROXY_USERNAME= +# WEBSHARE_PROXY_PASSWORD= +# WEBSHARE_PROXY_LOCATIONS=de,es diff --git a/README.md b/README.md index d420aab..744225b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,27 @@ Traefik übernimmt TLS via `myresolver`. Persistente Daten liegen in `./data/`. | `POST /api/recaps` | JWT | Recap erstellen | | `POST /api/recaps/{video_id}/regenerate` | JWT | Neu generieren | +## YouTube IP-Block / Proxy + +YouTube blockiert oft Server-IPs (Hetzner, AWS, …). Optional in `.env`: + +```bash +# Statischer Proxy (z.B. aus Webshare „Proxy Server“ / Dashboard-Zeile) +YOUTUBE_PROXY=http://USER:PASS@HOST:PORT +``` + +Für yt-dlp-Fallback und VTT-Downloads wird dieselbe URL genutzt. + +**Webshare-Hinweis:** Der kostenlose Tier und „Proxy Server“ (einzelne IP wie im Dashboard) funktionieren *manchmal* für wenige Recaps pro Tag — laut `youtube-transcript-api` aber unzuverlässig. Rotating **Residential** (kostenpflichtig) ist die robuste Lösung: + +```bash +WEBSHARE_PROXY_USERNAME=... +WEBSHARE_PROXY_PASSWORD=... +WEBSHARE_PROXY_LOCATIONS=de +``` + +Öffentliche Gratis-Proxy-Listen: nein (langsam, unsicher, meist schon gebannt). + ## Stack - FastAPI + SQLite diff --git a/compose.dev.yml b/compose.dev.yml index 34b6f32..ac42cb2 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -8,6 +8,10 @@ services: - JWT_SECRET=${JWT_SECRET:-dev-secret-change-me} - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} - OPENROUTER_MODEL=${OPENROUTER_MODEL:-google/gemini-2.5-flash} + - YOUTUBE_PROXY=${YOUTUBE_PROXY:-} + - WEBSHARE_PROXY_USERNAME=${WEBSHARE_PROXY_USERNAME:-} + - WEBSHARE_PROXY_PASSWORD=${WEBSHARE_PROXY_PASSWORD:-} + - WEBSHARE_PROXY_LOCATIONS=${WEBSHARE_PROXY_LOCATIONS:-} - DB_PATH=/data/ytrecap.sqlite - APP_URL=http://localhost:8000 volumes: diff --git a/compose.yml b/compose.yml index a110fe5..0dd7f8f 100644 --- a/compose.yml +++ b/compose.yml @@ -8,6 +8,10 @@ services: - JWT_EXPIRE_DAYS=${JWT_EXPIRE_DAYS:-90} - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} - OPENROUTER_MODEL=${OPENROUTER_MODEL:-google/gemini-2.5-flash} + - YOUTUBE_PROXY=${YOUTUBE_PROXY:-} + - WEBSHARE_PROXY_USERNAME=${WEBSHARE_PROXY_USERNAME:-} + - WEBSHARE_PROXY_PASSWORD=${WEBSHARE_PROXY_PASSWORD:-} + - WEBSHARE_PROXY_LOCATIONS=${WEBSHARE_PROXY_LOCATIONS:-} - DB_PATH=/data/ytrecap.sqlite - APP_URL=https://ytrecap.schwenk.online - TZ=Europe/Berlin diff --git a/server/settings.py b/server/settings.py index d699bc6..f0a4065 100644 --- a/server/settings.py +++ b/server/settings.py @@ -18,5 +18,13 @@ class Settings(BaseSettings): app_url: str = "http://localhost:8000" app_version: str = "1.0.0" + # YouTube transcript proxy (optional — for IP blocks) + # Single URL used for HTTP+HTTPS, e.g. http://user:pass@host:port + youtube_proxy: str = "" + # Webshare rotating residential (paid package — NOT free "Proxy Server" tier) + webshare_proxy_username: str = "" + webshare_proxy_password: str = "" + webshare_proxy_locations: str = "" # comma-separated, e.g. de,es + settings = Settings() # type: ignore[call-arg] diff --git a/server/transcripts.py b/server/transcripts.py index 5cc4213..53cf95f 100644 --- a/server/transcripts.py +++ b/server/transcripts.py @@ -4,14 +4,20 @@ import hashlib import logging import re from dataclasses import dataclass +from typing import Any from youtube_transcript_api import YouTubeTranscriptApi from youtube_transcript_api._errors import ( + IpBlocked, NoTranscriptFound, + RequestBlocked, TranscriptsDisabled, VideoUnavailable, YouTubeRequestFailed, ) +from youtube_transcript_api.proxies import GenericProxyConfig, ProxyConfig, WebshareProxyConfig + +from server.settings import settings logger = logging.getLogger(__name__) @@ -59,14 +65,75 @@ def _normalize_ytt_segments(raw: list[dict]) -> list[TranscriptSegment]: return [TranscriptSegment(start=float(item["start"]), text=str(item["text"])) for item in raw] +def _proxy_dict() -> dict[str, str] | None: + config = build_proxy_config() + if config is None: + return None + return config.to_requests_dict() + + +def build_proxy_config() -> ProxyConfig | None: + if settings.webshare_proxy_username and settings.webshare_proxy_password: + locations = [ + code.strip().lower() + for code in settings.webshare_proxy_locations.split(",") + if code.strip() + ] + return WebshareProxyConfig( + proxy_username=settings.webshare_proxy_username, + proxy_password=settings.webshare_proxy_password, + filter_ip_locations=locations or None, + ) + + proxy_url = settings.youtube_proxy.strip() + if proxy_url: + return GenericProxyConfig(http_url=proxy_url, https_url=proxy_url) + + return None + + +def proxy_url_for_ytdlp() -> str | None: + direct = settings.youtube_proxy.strip() + if direct: + return direct + config = build_proxy_config() + if config is None: + return None + proxies = config.to_requests_dict() + return proxies.get("https") or proxies.get("http") + + +def build_youtube_transcript_api() -> YouTubeTranscriptApi: + config = build_proxy_config() + if config is None: + return YouTubeTranscriptApi() + return YouTubeTranscriptApi(proxy_config=config) + + +def _blocked_message() -> str: + if build_proxy_config() is not None: + return ( + "YouTube blockiert auch über den Proxy. " + "Statische/datacenter Proxies halten oft nicht lange — " + "für zuverlässigere Ergebnisse rotating residential nutzen." + ) + return ( + "YouTube blockiert deine Server-IP. " + "Setze YOUTUBE_PROXY in der .env (siehe README)." + ) + + def _fetch_with_youtube_transcript_api(video_id: str) -> Transcript: - api = YouTubeTranscriptApi() + api = build_youtube_transcript_api() try: fetched = api.fetch(video_id, languages=list(PREFERRED_LANGUAGES)) segments = _normalize_ytt_segments( [{"start": s.start, "text": s.text} for s in fetched] ) return Transcript(language=fetched.language_code, segments=segments, source="youtube-transcript-api") + except (RequestBlocked, IpBlocked) as exc: + logger.warning("YouTube IP block for %s: %s", video_id, exc) + raise ValueError(_blocked_message()) from exc except (TranscriptsDisabled, NoTranscriptFound, VideoUnavailable): raise except YouTubeRequestFailed as exc: @@ -104,7 +171,7 @@ def _fetch_with_ytdlp(video_id: str) -> Transcript: import yt_dlp url = f"https://www.youtube.com/watch?v={video_id}" - opts: dict = { + opts: dict[str, Any] = { "skip_download": True, "quiet": True, "no_warnings": True, @@ -113,6 +180,10 @@ def _fetch_with_ytdlp(video_id: str) -> Transcript: "subtitleslangs": list(PREFERRED_LANGUAGES), "subtitlesformat": "vtt", } + proxy = proxy_url_for_ytdlp() + if proxy: + opts["proxy"] = proxy + with yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(url, download=False) @@ -145,9 +216,17 @@ def _fetch_with_ytdlp(video_id: str) -> Transcript: import httpx - resp = httpx.get(vtt_url, timeout=30.0, follow_redirects=True) - resp.raise_for_status() - segments = _parse_vtt(resp.text) + client_kwargs: dict[str, Any] = {"timeout": 30.0, "follow_redirects": True} + proxies = _proxy_dict() + if proxies: + client_kwargs["proxy"] = proxies.get("https") or proxies.get("http") + + with httpx.Client(**client_kwargs) as client: + resp = client.get(vtt_url) + resp.raise_for_status() + vtt_text = resp.text + + segments = _parse_vtt(vtt_text) if not segments: raise ValueError("Untertitel konnten nicht geparst werden.")