diff --git a/.env.example b/.env.example index 6c36c8a..7714907 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,14 @@ GOOGLE_CLIENT_SECRET= GOOGLE_REDIRECT_URI=https://testmyyoutube.vrubel.xyz/api/auth/google/callback ALLOWED_GOOGLE_EMAIL= +# Google/YouTube endpoints (необязательно, есть дефолты) +GOOGLE_AUTH_URI=https://accounts.google.com/o/oauth2/auth +GOOGLE_TOKEN_URI=https://oauth2.googleapis.com/token +GOOGLE_USERINFO_URI=https://www.googleapis.com/oauth2/v3/userinfo +GOOGLE_REVOKE_URI=https://oauth2.googleapis.com/revoke +YOUTUBE_API_BASE_URL=https://www.googleapis.com/youtube/v3 +YOUTUBE_WATCH_URL_TEMPLATE=https://www.youtube.com/watch?v={video_id} + METUBE_API_BASE_URL=http://192.168.8.177:8081 METUBE_PUBLIC_BASE_URL=http://192.168.8.177:8081 METUBE_CONTAINER_DOWNLOAD_DIR=/downloads diff --git a/Dockerfile b/Dockerfile index 94050c0..501999b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,11 @@ WORKDIR /app COPY entrypoint.sh ./entrypoint.sh RUN chmod +x ./entrypoint.sh +# Run as an unprivileged user: the app only reads the baked-in code/static +# assets and talks to Postgres/MeTube over the network, it never needs root. +RUN useradd --uid 10001 app +USER app + EXPOSE 8080 ENTRYPOINT ["./entrypoint.sh"] diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py index 8b293d3..a475d3c 100644 --- a/backend/app/api/auth.py +++ b/backend/app/api/auth.py @@ -17,11 +17,16 @@ router = APIRouter() @router.get("/auth/status") def auth_status(request: Request, db: Session = Depends(get_db)) -> dict: + # This endpoint is public (the Connect page needs `authenticated` before + # logging in), so the connected account's email must only be revealed to a + # request holding a valid session. + authenticated = bool(request.session.get("authenticated")) connected = google_oauth.is_connected(db) + email = google_oauth.get_connected_email(db) if authenticated and connected else None return { - "authenticated": bool(request.session.get("authenticated")), + "authenticated": authenticated, "connected": connected, - "email": google_oauth.get_connected_email(db) if connected else None, + "email": email, } diff --git a/backend/app/config.py b/backend/app/config.py index 3501206..d6e5d37 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -17,6 +17,15 @@ class Settings(BaseSettings): google_redirect_uri: str = "" allowed_google_email: str = "" + # Google/YouTube endpoints (AGENTS.md rule 12): configurable via env, + # defaults keep the current behaviour without touching .env. + google_auth_uri: str = "https://accounts.google.com/o/oauth2/auth" + google_token_uri: str = "https://oauth2.googleapis.com/token" + google_userinfo_uri: str = "https://www.googleapis.com/oauth2/v3/userinfo" + google_revoke_uri: str = "https://oauth2.googleapis.com/revoke" + youtube_api_base_url: str = "https://www.googleapis.com/youtube/v3" + youtube_watch_url_template: str = "https://www.youtube.com/watch?v={video_id}" + metube_api_base_url: str = "http://127.0.0.1:8081" metube_public_base_url: str = "http://127.0.0.1:8081" metube_container_download_dir: str = "/downloads" diff --git a/backend/app/models/oauth_credentials.py b/backend/app/models/oauth_credentials.py index 363786d..0467430 100644 --- a/backend/app/models/oauth_credentials.py +++ b/backend/app/models/oauth_credentials.py @@ -14,7 +14,6 @@ class OAuthCredentials(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True) google_email: Mapped[str] = mapped_column(String(255), nullable=False) encrypted_refresh_token: Mapped[str] = mapped_column(String, nullable=False) - access_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=False) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False diff --git a/backend/app/services/download_jobs.py b/backend/app/services/download_jobs.py index 422ee31..1b444ea 100644 --- a/backend/app/services/download_jobs.py +++ b/backend/app/services/download_jobs.py @@ -117,6 +117,26 @@ def _find_job_by_payload(db: Session, payload: dict) -> DownloadJob | None: return None +def _find_job_by_key(db: Session, key: str) -> DownloadJob | None: + """Match a bare MeTube store key (its 'canceled'/'cleared' events carry + just one key, JSON-string-encoded) to our latest job for it. The key is + the URL the download was enqueued under; matching the metube job id first + keeps us tolerant of both shapes.""" + job = ( + db.query(DownloadJob) + .filter(DownloadJob.metube_job_id == key) + .order_by(DownloadJob.id.desc()) + .first() + ) + if job is not None: + return job + + video = db.query(Video).filter(Video.youtube_url == key).one_or_none() + if video is not None: + return get_latest_job(db, video.id) + return None + + async def handle_metube_event(db: Session, event_name: str, raw_payload) -> None: try: payload = json.loads(raw_payload) if isinstance(raw_payload, str) else raw_payload @@ -124,21 +144,34 @@ async def handle_metube_event(db: Session, event_name: str, raw_payload) -> None logger.warning("Could not parse MeTube event payload for %s: %r", event_name, raw_payload) return - if event_name in ("canceled", "cleared"): + if event_name == "canceled": if not isinstance(payload, str): return - job = ( - db.query(DownloadJob) - .filter(DownloadJob.metube_job_id == payload) - .order_by(DownloadJob.id.desc()) - .first() - ) - if job is not None and event_name == "canceled" and job.status in ACTIVE_STATUSES: + job = _find_job_by_key(db, payload) + if job is not None and job.status in ACTIVE_STATUSES: job.status = "failed" job.error_message = "Отменено в MeTube" db.commit() return + if event_name == "cleared": + # MeTube emits 'cleared' ONLY when a *done* entry is deleted (trash + # via /delete?where=done, or CLEAR_COMPLETED_AFTER): the payload is + # that entry's key (its URL) as a JSON string, one event per item. + # Clearing the queue emits 'canceled' instead. The removed entry is + # usually a download MeTube had before we existed (AGENTS.md rule 10) + # or one we already completed, so an unmatched/empty payload must not + # touch anything: failing all active jobs here would wrongly mark + # downloads MeTube is still running. + if not isinstance(payload, str) or not payload: + return + job = _find_job_by_key(db, payload) + if job is not None and job.status in ACTIVE_STATUSES: + job.status = "failed" + job.error_message = "Очищено в MeTube" + db.commit() + return + if not isinstance(payload, dict): return diff --git a/backend/app/services/google_oauth.py b/backend/app/services/google_oauth.py index d22cd05..5de5b9d 100644 --- a/backend/app/services/google_oauth.py +++ b/backend/app/services/google_oauth.py @@ -1,6 +1,7 @@ import logging import os -from datetime import datetime, timezone +import threading +from datetime import datetime, timedelta, timezone # Google frequently echoes back a scope string that's a superset/reordering # of what we requested (e.g. we ask for "youtube", Google's token response @@ -34,10 +35,15 @@ SCOPES = [ "https://www.googleapis.com/auth/userinfo.profile", ] -AUTH_URI = "https://accounts.google.com/o/oauth2/auth" -TOKEN_URI = "https://oauth2.googleapis.com/token" -USERINFO_URI = "https://www.googleapis.com/oauth2/v3/userinfo" -REVOKE_URI = "https://oauth2.googleapis.com/revoke" +# Refresh proactively: never hand out a token that could die mid-request. +_ACCESS_TOKEN_REFRESH_BUFFER_SECONDS = 60 + +# Module-level access-token cache. Google access tokens live ~1h; previously +# every get_credentials() call (sync, unsubscribe, ...) paid for a full token +# exchange with Google. The lock keeps concurrent callers sharing one refresh +# instead of racing each other to the token endpoint. +_credentials_lock = threading.Lock() +_cached_credentials: Credentials | None = None class OAuthNotConnected(Exception): @@ -49,8 +55,8 @@ def _client_config() -> dict: "web": { "client_id": settings.google_client_id, "client_secret": settings.google_client_secret, - "auth_uri": AUTH_URI, - "token_uri": TOKEN_URI, + "auth_uri": settings.google_auth_uri, + "token_uri": settings.google_token_uri, "redirect_uris": [settings.google_redirect_uri], } } @@ -83,7 +89,7 @@ def exchange_code(code: str, state: str) -> Credentials: def fetch_userinfo(access_token: str) -> dict: response = httpx.get( - USERINFO_URI, + settings.google_userinfo_uri, headers={"Authorization": f"Bearer {access_token}"}, timeout=settings.metube_request_timeout_seconds, ) @@ -93,16 +99,14 @@ def fetch_userinfo(access_token: str) -> dict: def revoke_token(token: str) -> None: try: - httpx.post(REVOKE_URI, params={"token": token}, timeout=10) + httpx.post(settings.google_revoke_uri, params={"token": token}, timeout=10) except Exception: logger.warning("Failed to revoke Google token", exc_info=True) def store_credentials(db: Session, google_email: str, credentials: Credentials) -> None: + global _cached_credentials encrypted = encrypt_token(credentials.refresh_token) - expires_at = credentials.expiry - if expires_at is not None and expires_at.tzinfo is None: - expires_at = expires_at.replace(tzinfo=timezone.utc) row = db.get(OAuthCredentials, SINGLETON_ID) if row is None: @@ -111,15 +115,22 @@ def store_credentials(db: Session, google_email: str, credentials: Credentials) else: row.google_email = google_email row.encrypted_refresh_token = encrypted - row.access_token_expires_at = expires_at db.commit() + # The exchanged credentials carry a fresh access token -- reuse them so + # the immediately following syncs don't pay for a second Google request. + with _credentials_lock: + _cached_credentials = credentials + def clear_credentials(db: Session) -> None: + global _cached_credentials row = db.get(OAuthCredentials, SINGLETON_ID) if row is not None: db.delete(row) db.commit() + with _credentials_lock: + _cached_credentials = None def is_connected(db: Session) -> bool: @@ -131,26 +142,48 @@ def get_connected_email(db: Session) -> str | None: return row.google_email if row else None +def _token_missing_or_expiring(credentials: Credentials) -> bool: + if not credentials.token: + return True + expiry = credentials.expiry + if expiry is None: + # Unknown expiry -- be conservative and refresh. + return True + if expiry.tzinfo is None: + # google-auth reports expiry as a naive UTC datetime. + expiry = expiry.replace(tzinfo=timezone.utc) + return expiry <= datetime.now(timezone.utc) + timedelta(seconds=_ACCESS_TOKEN_REFRESH_BUFFER_SECONDS) + + def get_credentials(db: Session) -> Credentials: + global _cached_credentials row = db.get(OAuthCredentials, SINGLETON_ID) if row is None: raise OAuthNotConnected("Google account is not connected") refresh_token = decrypt_token(row.encrypted_refresh_token) - credentials = Credentials( - token=None, - refresh_token=refresh_token, - token_uri=TOKEN_URI, - client_id=settings.google_client_id, - client_secret=settings.google_client_secret, - scopes=SCOPES, - ) - credentials.refresh(GoogleAuthRequest()) - expires_at = credentials.expiry - if expires_at is not None and expires_at.tzinfo is None: - expires_at = expires_at.replace(tzinfo=timezone.utc) - row.access_token_expires_at = expires_at - db.commit() + with _credentials_lock: + credentials = _cached_credentials + if credentials is None or credentials.refresh_token != refresh_token: + # No cached token yet, or the account was reconnected with a new + # refresh token. The refresh below does the initial exchange. + credentials = Credentials( + token=None, + refresh_token=refresh_token, + token_uri=settings.google_token_uri, + client_id=settings.google_client_id, + client_secret=settings.google_client_secret, + scopes=SCOPES, + ) + _cached_credentials = credentials - return credentials + if _token_missing_or_expiring(credentials): + try: + credentials.refresh(GoogleAuthRequest()) + except Exception: + # Don't keep a broken cached object behind. + _cached_credentials = None + raise + + return credentials diff --git a/backend/app/services/metube_client.py b/backend/app/services/metube_client.py index 029a193..9d2a0f4 100644 --- a/backend/app/services/metube_client.py +++ b/backend/app/services/metube_client.py @@ -10,7 +10,10 @@ Verified against the real MeTube source (alexta69/metube, app/main.py + app/ytdl JSON-*string*-encoded (json.JSONEncoder().encode(...)), not a raw object — must json.loads() the payload. Relevant keys: id, title, url, status, msg, percent (float 0-100 or None), filename (already relative to DOWNLOAD_DIR), error. - 'canceled'/'cleared' carry just an id (also JSON-string-encoded). + 'canceled'/'cleared' carry the download's URL key (JSON-string-encoded), + one event per item; 'cleared' fires only when a done entry is deleted + (trash via /delete?where=done or CLEAR_COMPLETED_AFTER) — clearing the + queue emits 'canceled'. - MeTube status vocabulary: pending/preparing/scheduled/downloading/postprocessing/ finished/error — mapped to our own vocabulary in sync with download_jobs. - GET /history returns {"queue": [...], "pending": [...], "done": [...]} of the diff --git a/backend/app/services/sync.py b/backend/app/services/sync.py index 6405f89..c15778f 100644 --- a/backend/app/services/sync.py +++ b/backend/app/services/sync.py @@ -195,7 +195,7 @@ def sync_videos(db: Session) -> dict: continue duration_seconds = parse_iso8601_duration(item["duration_iso8601"]) - youtube_url = f"https://www.youtube.com/watch?v={item['youtube_video_id']}" + youtube_url = settings.youtube_watch_url_template.format(video_id=item["youtube_video_id"]) video = existing.get(item["youtube_video_id"]) if video is None: diff --git a/backend/app/services/youtube_client.py b/backend/app/services/youtube_client.py index 3223615..fc8aae0 100644 --- a/backend/app/services/youtube_client.py +++ b/backend/app/services/youtube_client.py @@ -7,7 +7,6 @@ from app.config import settings logger = logging.getLogger(__name__) -API_BASE = "https://www.googleapis.com/youtube/v3" BATCH_SIZE = 50 @@ -68,7 +67,7 @@ def fetch_subscriptions(credentials: Credentials) -> list[dict]: if page_token: params["pageToken"] = page_token - response = client.get(f"{API_BASE}/subscriptions", params=params, headers=_headers(credentials)) + response = client.get(f"{settings.youtube_api_base_url}/subscriptions", params=params, headers=_headers(credentials)) _raise_for_status(response) data = response.json() @@ -109,7 +108,7 @@ def fetch_playlist_video_ids(credentials: Credentials, playlist_id: str, max_res "playlistId": playlist_id, "maxResults": min(max_results, 50), } - response = client.get(f"{API_BASE}/playlistItems", params=params, headers=_headers(credentials)) + response = client.get(f"{settings.youtube_api_base_url}/playlistItems", params=params, headers=_headers(credentials)) if response.status_code == 404: return [] _raise_for_status(response) @@ -134,7 +133,7 @@ def fetch_videos_details(credentials: Credentials, video_ids: list[str]) -> list "id": ",".join(batch), "maxResults": BATCH_SIZE, } - response = client.get(f"{API_BASE}/videos", params=params, headers=_headers(credentials)) + response = client.get(f"{settings.youtube_api_base_url}/videos", params=params, headers=_headers(credentials)) _raise_for_status(response) data = response.json() @@ -162,7 +161,7 @@ def fetch_videos_details(credentials: Credentials, video_ids: list[str]) -> list def unsubscribe(credentials: Credentials, youtube_subscription_id: str) -> None: with httpx.Client(timeout=settings.metube_request_timeout_seconds) as client: response = client.delete( - f"{API_BASE}/subscriptions", + f"{settings.youtube_api_base_url}/subscriptions", params={"id": youtube_subscription_id}, headers=_headers(credentials), ) @@ -183,7 +182,7 @@ def fetch_uploads_playlists(credentials: Credentials, channel_ids: list[str]) -> "id": ",".join(batch), "maxResults": BATCH_SIZE, } - response = client.get(f"{API_BASE}/channels", params=params, headers=_headers(credentials)) + response = client.get(f"{settings.youtube_api_base_url}/channels", params=params, headers=_headers(credentials)) _raise_for_status(response) data = response.json() diff --git a/frontend/public/icons.svg b/frontend/public/icons.svg deleted file mode 100644 index e952219..0000000 --- a/frontend/public/icons.svg +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 564d242..cf106c9 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -13,7 +13,7 @@ export function authErrorMessage(code: string): string { async function request(path: string, options: RequestInit = {}): Promise { const res = await fetch(path, { credentials: 'include', - headers: { 'Content-Type': 'application/json', ...(options.headers ?? {}) }, + headers: options.body != null ? { 'Content-Type': 'application/json', ...(options.headers ?? {}) } : options.headers, ...options, }) if (!res.ok) { diff --git a/frontend/src/assets/hero.png b/frontend/src/assets/hero.png deleted file mode 100644 index 02251f4..0000000 Binary files a/frontend/src/assets/hero.png and /dev/null differ diff --git a/frontend/src/assets/react.svg b/frontend/src/assets/react.svg deleted file mode 100644 index 6c87de9..0000000 --- a/frontend/src/assets/react.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/frontend/src/assets/vite.svg b/frontend/src/assets/vite.svg deleted file mode 100644 index 5101b67..0000000 --- a/frontend/src/assets/vite.svg +++ /dev/null @@ -1 +0,0 @@ -Vite diff --git a/frontend/src/components/AppShell.tsx b/frontend/src/components/AppShell.tsx index 3c07ae4..f7bba3e 100644 --- a/frontend/src/components/AppShell.tsx +++ b/frontend/src/components/AppShell.tsx @@ -41,7 +41,6 @@ function Sidebar({ close }: { close: () => void }) { const categories = categoriesQuery.data ?? [] const links = [ { to: '/', icon: 'home' as const, text: 'Все видео', end: true }, - { to: '/uncategorized', icon: 'folder' as const, text: 'Без категории' }, { to: '/local', icon: 'server' as const, text: 'На сервере' }, ] return