Implement Phase 6: MeTube download integration
- MeTubeClient encapsulates all MeTube HTTP/Socket.IO calls (verified against
the real MeTube source: /add returns no job id, GET /history gives a queue
snapshot for reconciliation, filenames arrive already relative, percent is
a 0-100 float)
- download_jobs table + service: request/dedup active downloads, apply live
Socket.IO events (added/updated/completed/canceled/cleared) matched by
canonical YouTube URL, safe relative-path -> public media URL construction
- Reconciliation on startup against MeTube's live queue/done state (section 19):
non-terminal jobs recovered where possible, else marked "unknown"; already
completed jobs are left untouched
- POST/GET /api/videos/{id}/download(-status), recheck-local; feed/video
detail now report real local availability instead of a stub
- Frontend: download button with live status polling (queued/downloading %/
postprocessing/completed/failed+retry), local <video> playback with
YouTube fallback on playback error
- health.py now delegates to MeTubeClient (single place for MeTube calls)
26 new backend tests (63 total). Verified live: Socket.IO connects
successfully to the real MeTube instance on deploy.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
0ed20bb838
commit
fe16c08daa
27 changed files with 1125 additions and 48 deletions
|
|
@ -17,8 +17,9 @@ def client(monkeypatch, db_session):
|
|||
monkeypatch.setattr(
|
||||
google_oauth, "build_authorization_url", lambda: ("https://accounts.google.com/fake", "fixed-state")
|
||||
)
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
# Deliberately not using `with TestClient(app)`: that runs the app's
|
||||
# lifespan, which would try to reach the real MeTube instance and DB.
|
||||
yield TestClient(app)
|
||||
del app.dependency_overrides[get_db]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ def client(db_session):
|
|||
|
||||
app.dependency_overrides[get_db] = _get_db_override
|
||||
app.dependency_overrides[require_session] = lambda: None
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
# Deliberately not using `with TestClient(app)`: that runs the app's
|
||||
# lifespan, which would try to reach the real MeTube instance and DB.
|
||||
yield TestClient(app)
|
||||
del app.dependency_overrides[get_db]
|
||||
del app.dependency_overrides[require_session]
|
||||
|
||||
|
|
|
|||
243
tests/test_download_jobs.py
Normal file
243
tests/test_download_jobs.py
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models.channel import Channel
|
||||
from app.models.download_job import DownloadJob
|
||||
from app.models.video import Video
|
||||
from app.services import download_jobs
|
||||
|
||||
|
||||
def _seed_video(db_session, youtube_video_id="vid1", youtube_channel_id="chanA"):
|
||||
channel = Channel(youtube_channel_id=youtube_channel_id, title="Channel", subscribed=True)
|
||||
db_session.add(channel)
|
||||
db_session.commit()
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
video = Video(
|
||||
youtube_video_id=youtube_video_id,
|
||||
channel_id=channel.id,
|
||||
title="Video",
|
||||
published_at=datetime(2026, 9, 10, tzinfo=timezone.utc),
|
||||
youtube_url=f"https://www.youtube.com/watch?v={youtube_video_id}",
|
||||
)
|
||||
db_session.add(video)
|
||||
db_session.commit()
|
||||
db_session.refresh(video)
|
||||
return video
|
||||
|
||||
|
||||
def test_request_download_enqueues_and_creates_job(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
|
||||
calls = {}
|
||||
|
||||
def fake_enqueue(self, youtube_url, custom_name_prefix):
|
||||
calls["youtube_url"] = youtube_url
|
||||
calls["custom_name_prefix"] = custom_name_prefix
|
||||
return {"status": "ok"}
|
||||
|
||||
monkeypatch.setattr("app.services.metube_client.MeTubeClient.enqueue_video", fake_enqueue)
|
||||
|
||||
job = download_jobs.request_download(db_session, video)
|
||||
|
||||
assert job.status == "queued"
|
||||
assert calls["youtube_url"] == video.youtube_url
|
||||
assert calls["custom_name_prefix"] == video.youtube_video_id
|
||||
|
||||
|
||||
def test_request_download_is_idempotent_while_active(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.enqueue_video", lambda self, u, p: {"status": "ok"}
|
||||
)
|
||||
|
||||
job1 = download_jobs.request_download(db_session, video)
|
||||
job2 = download_jobs.request_download(db_session, video)
|
||||
|
||||
assert job1.id == job2.id
|
||||
assert db_session.query(DownloadJob).count() == 1
|
||||
|
||||
|
||||
def test_request_download_raises_on_metube_error(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.enqueue_video",
|
||||
lambda self, u, p: {"status": "error", "msg": "boom"},
|
||||
)
|
||||
|
||||
with pytest.raises(download_jobs.MeTubeRejected):
|
||||
download_jobs.request_download(db_session, video)
|
||||
|
||||
assert db_session.query(DownloadJob).count() == 0
|
||||
|
||||
|
||||
def test_request_download_allows_retry_after_failure(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="failed", error_message="oops")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.enqueue_video", lambda self, u, p: {"status": "ok"}
|
||||
)
|
||||
|
||||
new_job = download_jobs.request_download(db_session, video)
|
||||
assert new_job.id != job.id
|
||||
assert db_session.query(DownloadJob).count() == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_metube_event_added_and_updated(db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="queued")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
added_payload = json.dumps({"id": "vid1.vid1", "url": video.youtube_url, "status": "pending"})
|
||||
await download_jobs.handle_metube_event(db_session, "added", added_payload)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.metube_job_id == "vid1.vid1"
|
||||
assert job.status == "queued"
|
||||
|
||||
updating_payload = json.dumps(
|
||||
{"id": "vid1.vid1", "url": video.youtube_url, "status": "downloading", "percent": 42.5}
|
||||
)
|
||||
await download_jobs.handle_metube_event(db_session, "updated", updating_payload)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "downloading"
|
||||
assert job.progress_percent == 42
|
||||
assert job.started_at is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_metube_event_completed_builds_media_url(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="downloading", metube_job_id="vid1.vid1")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.build_media_url",
|
||||
lambda self, filename: f"http://metube.local/download/{filename}",
|
||||
)
|
||||
|
||||
payload = json.dumps(
|
||||
{"id": "vid1.vid1", "url": video.youtube_url, "status": "finished", "filename": "vid1.vid1.mp4"}
|
||||
)
|
||||
await download_jobs.handle_metube_event(db_session, "completed", payload)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "completed"
|
||||
assert job.progress_percent == 100
|
||||
assert job.metube_filename == "vid1.vid1.mp4"
|
||||
assert job.media_url == "http://metube.local/download/vid1.vid1.mp4"
|
||||
assert job.completed_at is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_metube_event_error_marks_failed(db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="downloading", metube_job_id="vid1.vid1")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
payload = json.dumps({"id": "vid1.vid1", "url": video.youtube_url, "status": "error", "msg": "network blip"})
|
||||
await download_jobs.handle_metube_event(db_session, "updated", payload)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "failed"
|
||||
assert job.error_message == "network blip"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_metube_event_ignores_unrelated_download(db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="queued")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
payload = json.dumps(
|
||||
{"id": "other.other", "url": "https://www.youtube.com/watch?v=other", "status": "finished"}
|
||||
)
|
||||
await download_jobs.handle_metube_event(db_session, "completed", payload)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "queued"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_metube_event_canceled_marks_failed(db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="downloading", metube_job_id="vid1.vid1")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
await download_jobs.handle_metube_event(db_session, "canceled", json.dumps("vid1.vid1"))
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "failed"
|
||||
|
||||
|
||||
def test_reconcile_marks_unknown_when_history_unavailable(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="downloading")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
def raise_error(self):
|
||||
raise RuntimeError("connection refused")
|
||||
|
||||
monkeypatch.setattr("app.services.metube_client.MeTubeClient.fetch_history", raise_error)
|
||||
|
||||
download_jobs.reconcile_on_startup(db_session)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "unknown"
|
||||
|
||||
|
||||
def test_reconcile_restores_completed_from_history(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="downloading")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.fetch_history",
|
||||
lambda self: {
|
||||
"queue": [],
|
||||
"pending": [],
|
||||
"done": [{"id": "vid1.vid1", "url": video.youtube_url, "status": "finished", "filename": "f.mp4"}],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.build_media_url",
|
||||
lambda self, filename: f"http://metube.local/download/{filename}",
|
||||
)
|
||||
|
||||
download_jobs.reconcile_on_startup(db_session)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "completed"
|
||||
assert job.media_url == "http://metube.local/download/f.mp4"
|
||||
|
||||
|
||||
def test_reconcile_leaves_completed_jobs_untouched(monkeypatch, db_session):
|
||||
video = _seed_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="completed", media_url="http://x/f.mp4")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
def fail_if_called(self):
|
||||
raise AssertionError("should not fetch history for completed jobs")
|
||||
|
||||
monkeypatch.setattr("app.services.metube_client.MeTubeClient.fetch_history", fail_if_called)
|
||||
|
||||
download_jobs.reconcile_on_startup(db_session)
|
||||
|
||||
db_session.refresh(job)
|
||||
assert job.status == "completed"
|
||||
assert job.media_url == "http://x/f.mp4"
|
||||
|
|
@ -19,8 +19,9 @@ def client(db_session):
|
|||
|
||||
app.dependency_overrides[get_db] = _get_db_override
|
||||
app.dependency_overrides[require_session] = lambda: None
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
# Deliberately not using `with TestClient(app)`: that runs the app's
|
||||
# lifespan, which would try to reach the real MeTube instance and DB.
|
||||
yield TestClient(app)
|
||||
del app.dependency_overrides[get_db]
|
||||
del app.dependency_overrides[require_session]
|
||||
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ def override_get_db():
|
|||
|
||||
|
||||
def test_health_ok():
|
||||
with patch("app.api.health.httpx.get") as mock_get:
|
||||
mock_get.return_value = MagicMock(status_code=200)
|
||||
with patch("app.services.metube_client.MeTubeClient.health", return_value=True):
|
||||
client = TestClient(app)
|
||||
response = client.get("/api/health")
|
||||
|
||||
|
|
@ -34,7 +33,7 @@ def test_health_ok():
|
|||
|
||||
|
||||
def test_health_metube_unreachable():
|
||||
with patch("app.api.health.httpx.get", side_effect=ConnectionError):
|
||||
with patch("app.services.metube_client.MeTubeClient.health", return_value=False):
|
||||
client = TestClient(app)
|
||||
response = client.get("/api/health")
|
||||
|
||||
|
|
|
|||
33
tests/test_metube_client.py
Normal file
33
tests/test_metube_client.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import pytest
|
||||
|
||||
from app.services.metube_client import MeTubeClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return MeTubeClient()
|
||||
|
||||
|
||||
def test_build_media_url_encodes_spaces(client):
|
||||
url = client.build_media_url("some file.mp4")
|
||||
assert url == f"{client.public_base_url}/download/some%20file.mp4"
|
||||
|
||||
|
||||
def test_build_media_url_strips_absolute_download_dir_prefix(client):
|
||||
filename = f"{client.download_dir}/nested/video.mp4"
|
||||
url = client.build_media_url(filename)
|
||||
assert url == f"{client.public_base_url}/download/nested/video.mp4"
|
||||
|
||||
|
||||
def test_build_media_url_rejects_path_traversal(client):
|
||||
assert client.build_media_url("../../etc/passwd") is None
|
||||
|
||||
|
||||
def test_build_media_url_none_for_empty(client):
|
||||
assert client.build_media_url("") is None
|
||||
assert client.build_media_url(None) is None
|
||||
|
||||
|
||||
def test_build_media_url_encodes_each_segment(client):
|
||||
url = client.build_media_url("dir with space/file#1.mp4")
|
||||
assert url == f"{client.public_base_url}/download/dir%20with%20space/file%231.mp4"
|
||||
|
|
@ -19,8 +19,10 @@ def client(db_session):
|
|||
|
||||
app.dependency_overrides[get_db] = _get_db_override
|
||||
app.dependency_overrides[require_session] = lambda: None
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
# Deliberately not using `with TestClient(app)`: that runs the app's
|
||||
# lifespan, which would try to reach the real MeTube instance and DB
|
||||
# (see AGENTS.md: MeTube network calls in tests must be opt-in only).
|
||||
yield TestClient(app)
|
||||
del app.dependency_overrides[get_db]
|
||||
del app.dependency_overrides[require_session]
|
||||
|
||||
|
|
@ -60,3 +62,99 @@ def test_get_video_by_youtube_id(client, db_session):
|
|||
def test_get_video_not_found(client):
|
||||
resp = client.get("/api/videos/missing")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def _seed_single_video(db_session, youtube_video_id="vid1"):
|
||||
channel = Channel(youtube_channel_id="chanA", title="Channel A", subscribed=True)
|
||||
db_session.add(channel)
|
||||
db_session.commit()
|
||||
|
||||
video = Video(
|
||||
youtube_video_id=youtube_video_id,
|
||||
channel_id=channel.id,
|
||||
title="Video One",
|
||||
published_at=datetime(2026, 9, 10, tzinfo=timezone.utc),
|
||||
youtube_url=f"https://www.youtube.com/watch?v={youtube_video_id}",
|
||||
)
|
||||
db_session.add(video)
|
||||
db_session.commit()
|
||||
db_session.refresh(video)
|
||||
return video
|
||||
|
||||
|
||||
def test_download_video_enqueues(client, db_session, monkeypatch):
|
||||
video = _seed_single_video(db_session)
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.enqueue_video", lambda self, u, p: {"status": "ok"}
|
||||
)
|
||||
|
||||
resp = client.post(f"/api/videos/{video.youtube_video_id}/download")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "queued"
|
||||
|
||||
|
||||
def test_download_video_metube_rejects(client, db_session, monkeypatch):
|
||||
video = _seed_single_video(db_session)
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.enqueue_video",
|
||||
lambda self, u, p: {"status": "error", "msg": "bad url"},
|
||||
)
|
||||
|
||||
resp = client.post(f"/api/videos/{video.youtube_video_id}/download")
|
||||
|
||||
assert resp.status_code == 502
|
||||
|
||||
|
||||
def test_download_video_second_click_returns_same_job(client, db_session, monkeypatch):
|
||||
video = _seed_single_video(db_session)
|
||||
monkeypatch.setattr(
|
||||
"app.services.metube_client.MeTubeClient.enqueue_video", lambda self, u, p: {"status": "ok"}
|
||||
)
|
||||
|
||||
r1 = client.post(f"/api/videos/{video.youtube_video_id}/download").json()
|
||||
r2 = client.post(f"/api/videos/{video.youtube_video_id}/download").json()
|
||||
|
||||
assert r1["requested_at"] == r2["requested_at"]
|
||||
|
||||
from app.models.download_job import DownloadJob
|
||||
|
||||
assert db_session.query(DownloadJob).count() == 1
|
||||
|
||||
|
||||
def test_download_status_not_downloaded(client, db_session):
|
||||
video = _seed_single_video(db_session)
|
||||
resp = client.get(f"/api/videos/{video.youtube_video_id}/download-status")
|
||||
assert resp.json()["status"] == "not_downloaded"
|
||||
|
||||
|
||||
def test_recheck_local_downgrades_to_unknown_when_media_missing(client, db_session, monkeypatch):
|
||||
from app.models.download_job import DownloadJob
|
||||
|
||||
video = _seed_single_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="completed", media_url="http://metube.local/download/f.mp4")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
monkeypatch.setattr("app.services.metube_client.MeTubeClient.check_media", lambda self, url: False)
|
||||
|
||||
resp = client.post(f"/api/videos/{video.youtube_video_id}/recheck-local")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "unknown"
|
||||
|
||||
|
||||
def test_recheck_local_keeps_completed_when_media_reachable(client, db_session, monkeypatch):
|
||||
from app.models.download_job import DownloadJob
|
||||
|
||||
video = _seed_single_video(db_session)
|
||||
job = DownloadJob(video_id=video.id, status="completed", media_url="http://metube.local/download/f.mp4")
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
monkeypatch.setattr("app.services.metube_client.MeTubeClient.check_media", lambda self, url: True)
|
||||
|
||||
resp = client.post(f"/api/videos/{video.youtube_video_id}/recheck-local")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "completed"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue