Fix review findings and remove Uncategorized from sidebar
Backend: cache Google access tokens (drop dead access_token_expires_at, migration 0007), handle MeTube cleared/canceled events by URL, return email from /auth/status only when authenticated, move Google base URLs into settings, run container as non-root. Frontend: include local feed filters in the query key, remove dead Saved page and unused assets, drop stale CategoryNav props and classes, send Content-Type only with a body, remove Uncategorized from sidebar.
This commit is contained in:
vrubelroman
parent
6c704cac97
commit
fde9a439df
25 changed files with 420 additions and 174 deletions
|
|
@ -117,6 +117,26 @@ def _find_job_by_payload(db: Session, payload: dict) -> DownloadJob | None:
|
|||
return None
|
||||
|
||||
|
||||
def _find_job_by_key(db: Session, key: str) -> DownloadJob | None:
|
||||
"""Match a bare MeTube store key (its 'canceled'/'cleared' events carry
|
||||
just one key, JSON-string-encoded) to our latest job for it. The key is
|
||||
the URL the download was enqueued under; matching the metube job id first
|
||||
keeps us tolerant of both shapes."""
|
||||
job = (
|
||||
db.query(DownloadJob)
|
||||
.filter(DownloadJob.metube_job_id == key)
|
||||
.order_by(DownloadJob.id.desc())
|
||||
.first()
|
||||
)
|
||||
if job is not None:
|
||||
return job
|
||||
|
||||
video = db.query(Video).filter(Video.youtube_url == key).one_or_none()
|
||||
if video is not None:
|
||||
return get_latest_job(db, video.id)
|
||||
return None
|
||||
|
||||
|
||||
async def handle_metube_event(db: Session, event_name: str, raw_payload) -> None:
|
||||
try:
|
||||
payload = json.loads(raw_payload) if isinstance(raw_payload, str) else raw_payload
|
||||
|
|
@ -124,21 +144,34 @@ async def handle_metube_event(db: Session, event_name: str, raw_payload) -> None
|
|||
logger.warning("Could not parse MeTube event payload for %s: %r", event_name, raw_payload)
|
||||
return
|
||||
|
||||
if event_name in ("canceled", "cleared"):
|
||||
if event_name == "canceled":
|
||||
if not isinstance(payload, str):
|
||||
return
|
||||
job = (
|
||||
db.query(DownloadJob)
|
||||
.filter(DownloadJob.metube_job_id == payload)
|
||||
.order_by(DownloadJob.id.desc())
|
||||
.first()
|
||||
)
|
||||
if job is not None and event_name == "canceled" and job.status in ACTIVE_STATUSES:
|
||||
job = _find_job_by_key(db, payload)
|
||||
if job is not None and job.status in ACTIVE_STATUSES:
|
||||
job.status = "failed"
|
||||
job.error_message = "Отменено в MeTube"
|
||||
db.commit()
|
||||
return
|
||||
|
||||
if event_name == "cleared":
|
||||
# MeTube emits 'cleared' ONLY when a *done* entry is deleted (trash
|
||||
# via /delete?where=done, or CLEAR_COMPLETED_AFTER): the payload is
|
||||
# that entry's key (its URL) as a JSON string, one event per item.
|
||||
# Clearing the queue emits 'canceled' instead. The removed entry is
|
||||
# usually a download MeTube had before we existed (AGENTS.md rule 10)
|
||||
# or one we already completed, so an unmatched/empty payload must not
|
||||
# touch anything: failing all active jobs here would wrongly mark
|
||||
# downloads MeTube is still running.
|
||||
if not isinstance(payload, str) or not payload:
|
||||
return
|
||||
job = _find_job_by_key(db, payload)
|
||||
if job is not None and job.status in ACTIVE_STATUSES:
|
||||
job.status = "failed"
|
||||
job.error_message = "Очищено в MeTube"
|
||||
db.commit()
|
||||
return
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
# Google frequently echoes back a scope string that's a superset/reordering
|
||||
# of what we requested (e.g. we ask for "youtube", Google's token response
|
||||
|
|
@ -34,10 +35,15 @@ SCOPES = [
|
|||
"https://www.googleapis.com/auth/userinfo.profile",
|
||||
]
|
||||
|
||||
AUTH_URI = "https://accounts.google.com/o/oauth2/auth"
|
||||
TOKEN_URI = "https://oauth2.googleapis.com/token"
|
||||
USERINFO_URI = "https://www.googleapis.com/oauth2/v3/userinfo"
|
||||
REVOKE_URI = "https://oauth2.googleapis.com/revoke"
|
||||
# Refresh proactively: never hand out a token that could die mid-request.
|
||||
_ACCESS_TOKEN_REFRESH_BUFFER_SECONDS = 60
|
||||
|
||||
# Module-level access-token cache. Google access tokens live ~1h; previously
|
||||
# every get_credentials() call (sync, unsubscribe, ...) paid for a full token
|
||||
# exchange with Google. The lock keeps concurrent callers sharing one refresh
|
||||
# instead of racing each other to the token endpoint.
|
||||
_credentials_lock = threading.Lock()
|
||||
_cached_credentials: Credentials | None = None
|
||||
|
||||
|
||||
class OAuthNotConnected(Exception):
|
||||
|
|
@ -49,8 +55,8 @@ def _client_config() -> dict:
|
|||
"web": {
|
||||
"client_id": settings.google_client_id,
|
||||
"client_secret": settings.google_client_secret,
|
||||
"auth_uri": AUTH_URI,
|
||||
"token_uri": TOKEN_URI,
|
||||
"auth_uri": settings.google_auth_uri,
|
||||
"token_uri": settings.google_token_uri,
|
||||
"redirect_uris": [settings.google_redirect_uri],
|
||||
}
|
||||
}
|
||||
|
|
@ -83,7 +89,7 @@ def exchange_code(code: str, state: str) -> Credentials:
|
|||
|
||||
def fetch_userinfo(access_token: str) -> dict:
|
||||
response = httpx.get(
|
||||
USERINFO_URI,
|
||||
settings.google_userinfo_uri,
|
||||
headers={"Authorization": f"Bearer {access_token}"},
|
||||
timeout=settings.metube_request_timeout_seconds,
|
||||
)
|
||||
|
|
@ -93,16 +99,14 @@ def fetch_userinfo(access_token: str) -> dict:
|
|||
|
||||
def revoke_token(token: str) -> None:
|
||||
try:
|
||||
httpx.post(REVOKE_URI, params={"token": token}, timeout=10)
|
||||
httpx.post(settings.google_revoke_uri, params={"token": token}, timeout=10)
|
||||
except Exception:
|
||||
logger.warning("Failed to revoke Google token", exc_info=True)
|
||||
|
||||
|
||||
def store_credentials(db: Session, google_email: str, credentials: Credentials) -> None:
|
||||
global _cached_credentials
|
||||
encrypted = encrypt_token(credentials.refresh_token)
|
||||
expires_at = credentials.expiry
|
||||
if expires_at is not None and expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=timezone.utc)
|
||||
|
||||
row = db.get(OAuthCredentials, SINGLETON_ID)
|
||||
if row is None:
|
||||
|
|
@ -111,15 +115,22 @@ def store_credentials(db: Session, google_email: str, credentials: Credentials)
|
|||
else:
|
||||
row.google_email = google_email
|
||||
row.encrypted_refresh_token = encrypted
|
||||
row.access_token_expires_at = expires_at
|
||||
db.commit()
|
||||
|
||||
# The exchanged credentials carry a fresh access token -- reuse them so
|
||||
# the immediately following syncs don't pay for a second Google request.
|
||||
with _credentials_lock:
|
||||
_cached_credentials = credentials
|
||||
|
||||
|
||||
def clear_credentials(db: Session) -> None:
|
||||
global _cached_credentials
|
||||
row = db.get(OAuthCredentials, SINGLETON_ID)
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
with _credentials_lock:
|
||||
_cached_credentials = None
|
||||
|
||||
|
||||
def is_connected(db: Session) -> bool:
|
||||
|
|
@ -131,26 +142,48 @@ def get_connected_email(db: Session) -> str | None:
|
|||
return row.google_email if row else None
|
||||
|
||||
|
||||
def _token_missing_or_expiring(credentials: Credentials) -> bool:
|
||||
if not credentials.token:
|
||||
return True
|
||||
expiry = credentials.expiry
|
||||
if expiry is None:
|
||||
# Unknown expiry -- be conservative and refresh.
|
||||
return True
|
||||
if expiry.tzinfo is None:
|
||||
# google-auth reports expiry as a naive UTC datetime.
|
||||
expiry = expiry.replace(tzinfo=timezone.utc)
|
||||
return expiry <= datetime.now(timezone.utc) + timedelta(seconds=_ACCESS_TOKEN_REFRESH_BUFFER_SECONDS)
|
||||
|
||||
|
||||
def get_credentials(db: Session) -> Credentials:
|
||||
global _cached_credentials
|
||||
row = db.get(OAuthCredentials, SINGLETON_ID)
|
||||
if row is None:
|
||||
raise OAuthNotConnected("Google account is not connected")
|
||||
|
||||
refresh_token = decrypt_token(row.encrypted_refresh_token)
|
||||
credentials = Credentials(
|
||||
token=None,
|
||||
refresh_token=refresh_token,
|
||||
token_uri=TOKEN_URI,
|
||||
client_id=settings.google_client_id,
|
||||
client_secret=settings.google_client_secret,
|
||||
scopes=SCOPES,
|
||||
)
|
||||
credentials.refresh(GoogleAuthRequest())
|
||||
|
||||
expires_at = credentials.expiry
|
||||
if expires_at is not None and expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=timezone.utc)
|
||||
row.access_token_expires_at = expires_at
|
||||
db.commit()
|
||||
with _credentials_lock:
|
||||
credentials = _cached_credentials
|
||||
if credentials is None or credentials.refresh_token != refresh_token:
|
||||
# No cached token yet, or the account was reconnected with a new
|
||||
# refresh token. The refresh below does the initial exchange.
|
||||
credentials = Credentials(
|
||||
token=None,
|
||||
refresh_token=refresh_token,
|
||||
token_uri=settings.google_token_uri,
|
||||
client_id=settings.google_client_id,
|
||||
client_secret=settings.google_client_secret,
|
||||
scopes=SCOPES,
|
||||
)
|
||||
_cached_credentials = credentials
|
||||
|
||||
return credentials
|
||||
if _token_missing_or_expiring(credentials):
|
||||
try:
|
||||
credentials.refresh(GoogleAuthRequest())
|
||||
except Exception:
|
||||
# Don't keep a broken cached object behind.
|
||||
_cached_credentials = None
|
||||
raise
|
||||
|
||||
return credentials
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ Verified against the real MeTube source (alexta69/metube, app/main.py + app/ytdl
|
|||
JSON-*string*-encoded (json.JSONEncoder().encode(...)), not a raw object — must
|
||||
json.loads() the payload. Relevant keys: id, title, url, status, msg, percent
|
||||
(float 0-100 or None), filename (already relative to DOWNLOAD_DIR), error.
|
||||
'canceled'/'cleared' carry just an id (also JSON-string-encoded).
|
||||
'canceled'/'cleared' carry the download's URL key (JSON-string-encoded),
|
||||
one event per item; 'cleared' fires only when a done entry is deleted
|
||||
(trash via /delete?where=done or CLEAR_COMPLETED_AFTER) — clearing the
|
||||
queue emits 'canceled'.
|
||||
- MeTube status vocabulary: pending/preparing/scheduled/downloading/postprocessing/
|
||||
finished/error — mapped to our own vocabulary in sync with download_jobs.
|
||||
- GET /history returns {"queue": [...], "pending": [...], "done": [...]} of the
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ def sync_videos(db: Session) -> dict:
|
|||
continue
|
||||
|
||||
duration_seconds = parse_iso8601_duration(item["duration_iso8601"])
|
||||
youtube_url = f"https://www.youtube.com/watch?v={item['youtube_video_id']}"
|
||||
youtube_url = settings.youtube_watch_url_template.format(video_id=item["youtube_video_id"])
|
||||
|
||||
video = existing.get(item["youtube_video_id"])
|
||||
if video is None:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from app.config import settings
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_BASE = "https://www.googleapis.com/youtube/v3"
|
||||
BATCH_SIZE = 50
|
||||
|
||||
|
||||
|
|
@ -68,7 +67,7 @@ def fetch_subscriptions(credentials: Credentials) -> list[dict]:
|
|||
if page_token:
|
||||
params["pageToken"] = page_token
|
||||
|
||||
response = client.get(f"{API_BASE}/subscriptions", params=params, headers=_headers(credentials))
|
||||
response = client.get(f"{settings.youtube_api_base_url}/subscriptions", params=params, headers=_headers(credentials))
|
||||
_raise_for_status(response)
|
||||
data = response.json()
|
||||
|
||||
|
|
@ -109,7 +108,7 @@ def fetch_playlist_video_ids(credentials: Credentials, playlist_id: str, max_res
|
|||
"playlistId": playlist_id,
|
||||
"maxResults": min(max_results, 50),
|
||||
}
|
||||
response = client.get(f"{API_BASE}/playlistItems", params=params, headers=_headers(credentials))
|
||||
response = client.get(f"{settings.youtube_api_base_url}/playlistItems", params=params, headers=_headers(credentials))
|
||||
if response.status_code == 404:
|
||||
return []
|
||||
_raise_for_status(response)
|
||||
|
|
@ -134,7 +133,7 @@ def fetch_videos_details(credentials: Credentials, video_ids: list[str]) -> list
|
|||
"id": ",".join(batch),
|
||||
"maxResults": BATCH_SIZE,
|
||||
}
|
||||
response = client.get(f"{API_BASE}/videos", params=params, headers=_headers(credentials))
|
||||
response = client.get(f"{settings.youtube_api_base_url}/videos", params=params, headers=_headers(credentials))
|
||||
_raise_for_status(response)
|
||||
data = response.json()
|
||||
|
||||
|
|
@ -162,7 +161,7 @@ def fetch_videos_details(credentials: Credentials, video_ids: list[str]) -> list
|
|||
def unsubscribe(credentials: Credentials, youtube_subscription_id: str) -> None:
|
||||
with httpx.Client(timeout=settings.metube_request_timeout_seconds) as client:
|
||||
response = client.delete(
|
||||
f"{API_BASE}/subscriptions",
|
||||
f"{settings.youtube_api_base_url}/subscriptions",
|
||||
params={"id": youtube_subscription_id},
|
||||
headers=_headers(credentials),
|
||||
)
|
||||
|
|
@ -183,7 +182,7 @@ def fetch_uploads_playlists(credentials: Credentials, channel_ids: list[str]) ->
|
|||
"id": ",".join(batch),
|
||||
"maxResults": BATCH_SIZE,
|
||||
}
|
||||
response = client.get(f"{API_BASE}/channels", params=params, headers=_headers(credentials))
|
||||
response = client.get(f"{settings.youtube_api_base_url}/channels", params=params, headers=_headers(credentials))
|
||||
_raise_for_status(response)
|
||||
data = response.json()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue