Implement Phase 6: MeTube download integration
- MeTubeClient encapsulates all MeTube HTTP/Socket.IO calls (verified against
the real MeTube source: /add returns no job id, GET /history gives a queue
snapshot for reconciliation, filenames arrive already relative, percent is
a 0-100 float)
- download_jobs table + service: request/dedup active downloads, apply live
Socket.IO events (added/updated/completed/canceled/cleared) matched by
canonical YouTube URL, safe relative-path -> public media URL construction
- Reconciliation on startup against MeTube's live queue/done state (section 19):
non-terminal jobs recovered where possible, else marked "unknown"; already
completed jobs are left untouched
- POST/GET /api/videos/{id}/download(-status), recheck-local; feed/video
detail now report real local availability instead of a stub
- Frontend: download button with live status polling (queued/downloading %/
postprocessing/completed/failed+retry), local <video> playback with
YouTube fallback on playback error
- health.py now delegates to MeTubeClient (single place for MeTube calls)
26 new backend tests (63 total). Verified live: Socket.IO connects
successfully to the real MeTube instance on deploy.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
0ed20bb838
commit
fe16c08daa
27 changed files with 1125 additions and 48 deletions
179
backend/app/services/download_jobs.py
Normal file
179
backend/app/services/download_jobs.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.download_job import ACTIVE_STATUSES, DownloadJob
|
||||
from app.models.video import Video
|
||||
from app.services.metube_client import METUBE_STATUS_MAP, MeTubeClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MeTubeRejected(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_latest_job(db: Session, video_id: int) -> DownloadJob | None:
|
||||
return (
|
||||
db.query(DownloadJob)
|
||||
.filter(DownloadJob.video_id == video_id)
|
||||
.order_by(DownloadJob.requested_at.desc(), DownloadJob.id.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def latest_jobs_map(db: Session, video_ids: list[int]) -> dict[int, DownloadJob]:
|
||||
if not video_ids:
|
||||
return {}
|
||||
rows = (
|
||||
db.query(DownloadJob)
|
||||
.filter(DownloadJob.video_id.in_(video_ids))
|
||||
.order_by(DownloadJob.requested_at.desc(), DownloadJob.id.desc())
|
||||
.all()
|
||||
)
|
||||
result: dict[int, DownloadJob] = {}
|
||||
for row in rows:
|
||||
result.setdefault(row.video_id, row)
|
||||
return result
|
||||
|
||||
|
||||
def request_download(db: Session, video: Video) -> DownloadJob:
|
||||
existing = get_latest_job(db, video.id)
|
||||
if existing is not None and existing.status in ACTIVE_STATUSES:
|
||||
return existing
|
||||
|
||||
result = MeTubeClient().enqueue_video(video.youtube_url, video.youtube_video_id)
|
||||
if result.get("status") == "error":
|
||||
raise MeTubeRejected(result.get("msg") or "MeTube rejected the download")
|
||||
|
||||
job = DownloadJob(video_id=video.id, status="queued")
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def _find_job_by_payload(db: Session, payload: dict) -> DownloadJob | None:
|
||||
metube_id = payload.get("id")
|
||||
if metube_id:
|
||||
job = (
|
||||
db.query(DownloadJob)
|
||||
.filter(DownloadJob.metube_job_id == metube_id)
|
||||
.order_by(DownloadJob.id.desc())
|
||||
.first()
|
||||
)
|
||||
if job is not None:
|
||||
return job
|
||||
|
||||
url = payload.get("url")
|
||||
if url:
|
||||
video = db.query(Video).filter(Video.youtube_url == url).one_or_none()
|
||||
if video is not None:
|
||||
job = get_latest_job(db, video.id)
|
||||
if job is not None:
|
||||
return job
|
||||
return None
|
||||
|
||||
|
||||
async def handle_metube_event(db: Session, event_name: str, raw_payload) -> None:
|
||||
try:
|
||||
payload = json.loads(raw_payload) if isinstance(raw_payload, str) else raw_payload
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Could not parse MeTube event payload for %s: %r", event_name, raw_payload)
|
||||
return
|
||||
|
||||
if event_name in ("canceled", "cleared"):
|
||||
if not isinstance(payload, str):
|
||||
return
|
||||
job = (
|
||||
db.query(DownloadJob)
|
||||
.filter(DownloadJob.metube_job_id == payload)
|
||||
.order_by(DownloadJob.id.desc())
|
||||
.first()
|
||||
)
|
||||
if job is not None and event_name == "canceled" and job.status in ACTIVE_STATUSES:
|
||||
job.status = "failed"
|
||||
job.error_message = "Отменено в MeTube"
|
||||
db.commit()
|
||||
return
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
|
||||
job = _find_job_by_payload(db, payload)
|
||||
if job is None:
|
||||
return
|
||||
|
||||
_apply_metube_info(job, payload)
|
||||
db.commit()
|
||||
logger.info("download_job %s updated to status=%s via '%s' event", job.id, job.status, event_name)
|
||||
|
||||
|
||||
def _apply_metube_info(job: DownloadJob, info: dict) -> None:
|
||||
if info.get("id"):
|
||||
job.metube_job_id = info["id"]
|
||||
|
||||
our_status = METUBE_STATUS_MAP.get(info.get("status"), "unknown")
|
||||
|
||||
if job.started_at is None and our_status in ("downloading", "postprocessing", "completed"):
|
||||
job.started_at = datetime.now(timezone.utc)
|
||||
|
||||
percent = info.get("percent")
|
||||
if isinstance(percent, (int, float)):
|
||||
job.progress_percent = int(percent)
|
||||
|
||||
if our_status == "failed":
|
||||
job.status = "failed"
|
||||
job.error_message = info.get("msg") or info.get("error") or "Ошибка загрузки"
|
||||
elif our_status == "completed":
|
||||
filename = info.get("filename")
|
||||
if filename:
|
||||
job.metube_filename = filename
|
||||
job.media_url = MeTubeClient().build_media_url(filename)
|
||||
job.status = "completed"
|
||||
job.progress_percent = 100
|
||||
job.completed_at = job.completed_at or datetime.now(timezone.utc)
|
||||
else:
|
||||
job.status = our_status
|
||||
|
||||
|
||||
def reconcile_on_startup(db: Session) -> None:
|
||||
"""Section 19: after a restart we don't trust in-flight jobs until we've
|
||||
checked MeTube's current state. Completed jobs with a media_url are left
|
||||
alone — they don't need MeTube's history to remain trustworthy."""
|
||||
stale_statuses = list(ACTIVE_STATUSES) + ["unknown"]
|
||||
jobs = db.query(DownloadJob).filter(DownloadJob.status.in_(stale_statuses)).all()
|
||||
if not jobs:
|
||||
return
|
||||
|
||||
client = MeTubeClient()
|
||||
try:
|
||||
history = client.fetch_history()
|
||||
except Exception:
|
||||
logger.warning("Could not fetch MeTube history for reconciliation", exc_info=True)
|
||||
for job in jobs:
|
||||
job.status = "unknown"
|
||||
db.commit()
|
||||
return
|
||||
|
||||
by_url: dict[str, dict] = {}
|
||||
for bucket in ("queue", "pending", "done"):
|
||||
for item in history.get(bucket, []) or []:
|
||||
url = item.get("url")
|
||||
if url:
|
||||
by_url[url] = item
|
||||
|
||||
videos = {v.id: v for v in db.query(Video).filter(Video.id.in_([j.video_id for j in jobs])).all()}
|
||||
|
||||
for job in jobs:
|
||||
video = videos.get(job.video_id)
|
||||
info = by_url.get(video.youtube_url) if video else None
|
||||
if info is None:
|
||||
job.status = "unknown"
|
||||
continue
|
||||
_apply_metube_info(job, info)
|
||||
|
||||
db.commit()
|
||||
logger.info("Reconciled %d download job(s) against MeTube history", len(jobs))
|
||||
Loading…
Add table
Add a link
Reference in a new issue