Channel features: subscribers, new-videos badges, activity-driven sync
- Store subscriber count from YouTube statistics and show it on channel page - Sync 50 videos per channel with playlistItems pagination support - Show per-channel and per-category new-videos counters (2-day window) - Replace hourly videos sync with activity trigger (2h idle) and incremental backfill (hard cap 200 per channel) - Clicking the sidebar new-videos count filters the category feed to recent videos only (new_only) - Update agent-team docs: deploy after green checks
This commit is contained in:
vrubelroman
parent
fde9a439df
commit
e10df8dcbd
27 changed files with 1420 additions and 107 deletions
|
|
@ -1,3 +1,6 @@
|
|||
from datetime import datetime, timezone
|
||||
|
||||
from app.config import settings
|
||||
from app.models.channel import Channel
|
||||
from app.models.video import Video
|
||||
from app.services import sync
|
||||
|
|
@ -32,7 +35,7 @@ def test_sync_subscriptions_idempotent_and_unsubscribes(monkeypatch, db_session)
|
|||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_uploads_playlists",
|
||||
lambda creds, ids: {cid: f"UU{cid}" for cid in ids},
|
||||
lambda creds, ids: {cid: {"uploads_playlist_id": f"UU{cid}", "subscriber_count": 12345} for cid in ids},
|
||||
)
|
||||
|
||||
result = sync.sync_subscriptions(db_session)
|
||||
|
|
@ -45,6 +48,7 @@ def test_sync_subscriptions_idempotent_and_unsubscribes(monkeypatch, db_session)
|
|||
assert [c.youtube_channel_id for c in channels] == ["chanA", "chanB"]
|
||||
assert all(c.subscribed for c in channels)
|
||||
assert channels[0].uploads_playlist_id == "UUchanA"
|
||||
assert channels[0].subscriber_count == 12345
|
||||
|
||||
# Second sync: chanA disappears from subscriptions, chanC appears.
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -81,6 +85,46 @@ def test_sync_subscriptions_idempotent_and_unsubscribes(monkeypatch, db_session)
|
|||
assert channels["chanC"].subscribed is True
|
||||
|
||||
|
||||
def test_sync_subscriptions_skips_none_subscriber_count(monkeypatch, db_session):
|
||||
channel = Channel(
|
||||
youtube_channel_id="chanA",
|
||||
youtube_subscription_id="subA",
|
||||
title="Channel A",
|
||||
subscribed=True,
|
||||
subscriber_count=12345,
|
||||
)
|
||||
db_session.add(channel)
|
||||
db_session.commit()
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: _fake_credentials())
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_subscriptions",
|
||||
lambda creds: [
|
||||
{
|
||||
"youtube_channel_id": "chanA",
|
||||
"youtube_subscription_id": "subA",
|
||||
"title": "Channel A",
|
||||
"description": "d",
|
||||
"thumbnail_url": "t",
|
||||
},
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_uploads_playlists",
|
||||
lambda creds, ids: {cid: {"uploads_playlist_id": f"UU{cid}", "subscriber_count": None} for cid in ids},
|
||||
)
|
||||
|
||||
result = sync.sync_subscriptions(db_session)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
db_session.refresh(channel)
|
||||
assert channel.uploads_playlist_id == "UUchanA"
|
||||
# None in the API response must not overwrite the previously stored count.
|
||||
assert channel.subscriber_count == 12345
|
||||
|
||||
|
||||
def test_sync_in_progress_raises(monkeypatch, db_session):
|
||||
sync._subscriptions_lock.acquire()
|
||||
try:
|
||||
|
|
@ -106,12 +150,14 @@ def _seed_channel(db_session, youtube_channel_id="chanA", uploads_playlist_id="U
|
|||
return channel
|
||||
|
||||
|
||||
def test_sync_videos_adds_and_updates(monkeypatch, db_session):
|
||||
def test_sync_videos_adds_new_videos(monkeypatch, db_session):
|
||||
channel = _seed_channel(db_session)
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: object())
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client, "fetch_playlist_video_ids", lambda creds, playlist_id, max_results: ["vid1"]
|
||||
sync.youtube_client,
|
||||
"fetch_playlist_video_ids_incremental",
|
||||
lambda creds, playlist_id, known_ids, max_results, stop_threshold: ["vid1"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
|
|
@ -133,12 +179,23 @@ def test_sync_videos_adds_and_updates(monkeypatch, db_session):
|
|||
|
||||
assert result["status"] == "completed"
|
||||
assert result["videos_added"] == 1
|
||||
assert result["channels_checked"] == 1
|
||||
|
||||
video = db_session.query(Video).filter_by(youtube_video_id="vid1").one()
|
||||
assert video.title == "Video One"
|
||||
assert video.duration_seconds == 300
|
||||
assert video.youtube_url == "https://www.youtube.com/watch?v=vid1"
|
||||
|
||||
|
||||
def test_sync_videos_second_run_is_idempotent_and_fetches_no_details(monkeypatch, db_session):
|
||||
channel = _seed_channel(db_session)
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: object())
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_playlist_video_ids_incremental",
|
||||
lambda creds, playlist_id, known_ids, max_results, stop_threshold: ["vid1"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
|
|
@ -146,29 +203,142 @@ def test_sync_videos_adds_and_updates(monkeypatch, db_session):
|
|||
{
|
||||
"youtube_video_id": "vid1",
|
||||
"youtube_channel_id": channel.youtube_channel_id,
|
||||
"title": "Video One Updated",
|
||||
"description": "d2",
|
||||
"thumbnail_url": "t2",
|
||||
"title": "Video One",
|
||||
"description": "d",
|
||||
"thumbnail_url": "t",
|
||||
"published_at": "2026-09-10T12:00:00Z",
|
||||
"duration_iso8601": "PT6M",
|
||||
"duration_iso8601": "PT5M",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
result2 = sync.sync_videos(db_session)
|
||||
assert result2["videos_added"] == 0
|
||||
assert result2["videos_updated"] == 1
|
||||
first = sync.sync_videos(db_session)
|
||||
assert first["videos_added"] == 1
|
||||
|
||||
videos = db_session.query(Video).all()
|
||||
assert len(videos) == 1
|
||||
assert videos[0].title == "Video One Updated"
|
||||
# Second run: vid1 is now known, so the incremental fetch reports nothing
|
||||
# new and no details are requested at all (existing videos are not
|
||||
# metadata-refreshed by design).
|
||||
details_calls = []
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_playlist_video_ids_incremental",
|
||||
lambda creds, playlist_id, known_ids, max_results, stop_threshold: [],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
lambda creds, ids: details_calls.append(list(ids)) or [],
|
||||
)
|
||||
|
||||
second = sync.sync_videos(db_session)
|
||||
|
||||
assert second["videos_added"] == 0
|
||||
assert second["videos_updated"] == 0
|
||||
assert details_calls == [[]]
|
||||
assert db_session.query(Video).count() == 1
|
||||
|
||||
|
||||
def test_sync_videos_passes_known_ids_and_backfill_settings(monkeypatch, db_session):
|
||||
channel = _seed_channel(db_session)
|
||||
db_session.add_all(
|
||||
[
|
||||
Video(
|
||||
youtube_video_id="known1",
|
||||
channel_id=channel.id,
|
||||
title="Known 1",
|
||||
published_at=datetime(2026, 9, 1, tzinfo=timezone.utc),
|
||||
youtube_url="https://www.youtube.com/watch?v=known1",
|
||||
),
|
||||
Video(
|
||||
youtube_video_id="known2",
|
||||
channel_id=channel.id,
|
||||
title="Known 2",
|
||||
published_at=datetime(2026, 9, 1, tzinfo=timezone.utc),
|
||||
youtube_url="https://www.youtube.com/watch?v=known2",
|
||||
),
|
||||
]
|
||||
)
|
||||
db_session.commit()
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_incremental(creds, playlist_id, known_ids, max_results, stop_threshold):
|
||||
captured.update(known_ids=known_ids, max_results=max_results, stop_threshold=stop_threshold)
|
||||
return ["vid1"]
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: object())
|
||||
monkeypatch.setattr(sync.youtube_client, "fetch_playlist_video_ids_incremental", fake_incremental)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
lambda creds, ids: [
|
||||
{
|
||||
"youtube_video_id": "vid1",
|
||||
"youtube_channel_id": channel.youtube_channel_id,
|
||||
"title": "Video One",
|
||||
"description": "d",
|
||||
"thumbnail_url": "t",
|
||||
"published_at": "2026-09-10T12:00:00Z",
|
||||
"duration_iso8601": "PT5M",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
result = sync.sync_videos(db_session)
|
||||
|
||||
assert result["videos_added"] == 1
|
||||
assert captured["known_ids"] == {"known1", "known2"}
|
||||
assert captured["max_results"] == settings.videos_backfill_cap
|
||||
assert captured["stop_threshold"] == settings.videos_known_stop_threshold
|
||||
|
||||
|
||||
def test_sync_videos_backfill_cap_ingests_all_candidates(monkeypatch, db_session):
|
||||
channel = _seed_channel(db_session)
|
||||
cap = settings.videos_backfill_cap
|
||||
new_ids = [f"vid{i}" for i in range(cap)]
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: object())
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_playlist_video_ids_incremental",
|
||||
lambda creds, playlist_id, known_ids, max_results, stop_threshold: new_ids,
|
||||
)
|
||||
details_calls = []
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
lambda creds, ids: details_calls.append(list(ids))
|
||||
or [
|
||||
{
|
||||
"youtube_video_id": vid,
|
||||
"youtube_channel_id": channel.youtube_channel_id,
|
||||
"title": f"Title {vid}",
|
||||
"description": "d",
|
||||
"thumbnail_url": "t",
|
||||
"published_at": "2026-09-10T12:00:00Z",
|
||||
"duration_iso8601": None,
|
||||
}
|
||||
for vid in ids
|
||||
],
|
||||
)
|
||||
|
||||
result = sync.sync_videos(db_session)
|
||||
|
||||
assert result["videos_added"] == cap
|
||||
assert result["videos_updated"] == 0
|
||||
assert set(details_calls[0]) == set(new_ids)
|
||||
assert db_session.query(Video).count() == cap
|
||||
|
||||
|
||||
def test_sync_videos_skips_unknown_channel(monkeypatch, db_session):
|
||||
_seed_channel(db_session, "chanA")
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: object())
|
||||
monkeypatch.setattr(sync.youtube_client, "fetch_playlist_video_ids", lambda creds, playlist_id, max_results: [])
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_playlist_video_ids_incremental",
|
||||
lambda creds, playlist_id, known_ids, max_results, stop_threshold: [],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
|
|
@ -189,3 +359,211 @@ def test_sync_videos_skips_unknown_channel(monkeypatch, db_session):
|
|||
|
||||
assert result["videos_skipped"] == 1
|
||||
assert db_session.query(Video).count() == 0
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload, status_code=200):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
self.text = str(payload)
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeCredentials:
|
||||
token = "fake-token"
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
"""Serves canned playlistItems pages through the real incremental fetcher."""
|
||||
|
||||
def __init__(self, pages):
|
||||
self._pages = list(pages)
|
||||
self.requests = []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def get(self, url, params=None, headers=None):
|
||||
self.requests.append({"url": url, "params": params})
|
||||
return self._pages.pop(0)
|
||||
|
||||
|
||||
def _page(items, next_token=None):
|
||||
return _FakeResponse(
|
||||
{
|
||||
"items": [{"contentDetails": {"videoId": vid}} for vid in items],
|
||||
**({"nextPageToken": next_token} if next_token else {}),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_sync_videos_stops_pagination_on_known_run(monkeypatch, db_session):
|
||||
"""A channel with many known videos in a row after the new ones: the
|
||||
incremental fetch must stop paginating once the known-run threshold is hit
|
||||
and only the new ids must get details."""
|
||||
channel = _seed_channel(db_session)
|
||||
known = [f"known{i}" for i in range(60)]
|
||||
db_session.add_all(
|
||||
[
|
||||
Video(
|
||||
youtube_video_id=vid,
|
||||
channel_id=channel.id,
|
||||
title=f"Known {vid}",
|
||||
published_at=datetime(2026, 9, 1, tzinfo=timezone.utc),
|
||||
youtube_url=f"https://www.youtube.com/watch?v={vid}",
|
||||
)
|
||||
for vid in known
|
||||
]
|
||||
)
|
||||
db_session.commit()
|
||||
|
||||
# Page 1: two new videos then 48 known; page 2 continues with known ids,
|
||||
# so the run crosses 50 on the second page and pagination must stop there
|
||||
# (a third page exists and must never be requested).
|
||||
client = _FakeClient(
|
||||
[
|
||||
_page(["new1", "new2", *known[:48]], next_token="page2"),
|
||||
_page(known[48:58], next_token="page3"),
|
||||
_page(["never-seen"]),
|
||||
]
|
||||
)
|
||||
details_calls = []
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: _FakeCredentials())
|
||||
monkeypatch.setattr(sync.youtube_client.httpx, "Client", lambda timeout: client)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
lambda creds, ids: details_calls.append(list(ids))
|
||||
or [
|
||||
{
|
||||
"youtube_video_id": vid,
|
||||
"youtube_channel_id": channel.youtube_channel_id,
|
||||
"title": f"Title {vid}",
|
||||
"description": "",
|
||||
"thumbnail_url": None,
|
||||
"published_at": "2026-09-10T12:00:00Z",
|
||||
"duration_iso8601": None,
|
||||
}
|
||||
for vid in ids
|
||||
],
|
||||
)
|
||||
|
||||
result = sync.sync_videos(db_session)
|
||||
|
||||
assert result["videos_added"] == 2
|
||||
assert len(client.requests) == 2 # stopped on page 2, page 3 never fetched
|
||||
assert set(details_calls[0]) == {"new1", "new2"}
|
||||
assert db_session.query(Video).count() == 62
|
||||
|
||||
|
||||
def test_sync_videos_stops_at_backfill_cap(monkeypatch, db_session):
|
||||
"""More new videos than the cap: pagination stops once the cap is reached
|
||||
and details are fetched for exactly the capped candidates."""
|
||||
channel = _seed_channel(db_session)
|
||||
cap = settings.videos_backfill_cap
|
||||
pages = [_page([f"vid{page * 50 + i}" for i in range(50)], next_token=f"p{page + 2}") for page in range(6)]
|
||||
client = _FakeClient(pages)
|
||||
details_calls = []
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: _FakeCredentials())
|
||||
monkeypatch.setattr(sync.youtube_client.httpx, "Client", lambda timeout: client)
|
||||
monkeypatch.setattr(
|
||||
sync.youtube_client,
|
||||
"fetch_videos_details",
|
||||
lambda creds, ids: details_calls.append(list(ids))
|
||||
or [
|
||||
{
|
||||
"youtube_video_id": vid,
|
||||
"youtube_channel_id": channel.youtube_channel_id,
|
||||
"title": f"Title {vid}",
|
||||
"description": "",
|
||||
"thumbnail_url": None,
|
||||
"published_at": "2026-09-10T12:00:00Z",
|
||||
"duration_iso8601": None,
|
||||
}
|
||||
for vid in ids
|
||||
],
|
||||
)
|
||||
|
||||
result = sync.sync_videos(db_session)
|
||||
|
||||
assert len(client.requests) == 4 # 4 pages x 50 = cap reached
|
||||
assert result["videos_added"] == cap
|
||||
assert len(details_calls[0]) == cap
|
||||
assert db_session.query(Video).count() == cap
|
||||
|
||||
|
||||
def test_sync_videos_cap_is_hard_history_depth_limit(monkeypatch, db_session):
|
||||
"""videos_backfill_cap is an intentional history-depth limit, not a
|
||||
per-sync batch: a 250-video channel ingests only the newest 200 videos
|
||||
ever; a later sync without new videos stops after one page of known ids
|
||||
and adds nothing; a new video on top of the playlist is picked up while
|
||||
the beyond-the-cap history never surfaces."""
|
||||
channel = _seed_channel(db_session)
|
||||
cap = settings.videos_backfill_cap
|
||||
|
||||
def _details(creds, ids):
|
||||
return [
|
||||
{
|
||||
"youtube_video_id": vid,
|
||||
"youtube_channel_id": channel.youtube_channel_id,
|
||||
"title": f"Title {vid}",
|
||||
"description": "",
|
||||
"thumbnail_url": None,
|
||||
"published_at": "2026-09-10T12:00:00Z",
|
||||
"duration_iso8601": None,
|
||||
}
|
||||
for vid in ids
|
||||
]
|
||||
|
||||
monkeypatch.setattr(sync.google_oauth, "get_credentials", lambda db: _FakeCredentials())
|
||||
monkeypatch.setattr(sync.youtube_client, "fetch_videos_details", _details)
|
||||
|
||||
# Uploads playlist, newest first: vid249 .. vid0.
|
||||
playlist = [f"vid{i}" for i in range(249, -1, -1)]
|
||||
|
||||
def _run(ids):
|
||||
client = _FakeClient(
|
||||
[_page(ids[i : i + 50], next_token=f"page{i // 50}") for i in range(0, len(ids), 50)]
|
||||
)
|
||||
monkeypatch.setattr(sync.youtube_client.httpx, "Client", lambda timeout: client)
|
||||
return sync.sync_videos(db_session), client
|
||||
|
||||
# First sync: 250 videos, only the newest 200 fit under the cap.
|
||||
result1, client1 = _run(playlist)
|
||||
assert result1["videos_added"] == cap
|
||||
assert db_session.query(Video).count() == cap
|
||||
assert len(client1.requests) == 4 # 4 pages x 50 = cap, older pages untouched
|
||||
synced = {v.youtube_video_id for v in db_session.query(Video).all()}
|
||||
assert synced == {f"vid{i}" for i in range(50, 250)}
|
||||
assert "vid49" not in synced # older than the cap: never backfilled by design
|
||||
|
||||
# Second sync, nothing new: one page of 50 known ids in a row stops it.
|
||||
result2, client2 = _run(playlist)
|
||||
assert result2["videos_added"] == 0
|
||||
assert len(client2.requests) == 1
|
||||
assert db_session.query(Video).count() == cap
|
||||
|
||||
# Third sync, one new video on top: only that one is added, the
|
||||
# beyond-the-cap history still does not surface.
|
||||
result3, client3 = _run(["vid250", *playlist])
|
||||
assert result3["videos_added"] == 1
|
||||
assert len(client3.requests) == 2 # page 2 needed to confirm 50 known in a row
|
||||
synced_after = {v.youtube_video_id for v in db_session.query(Video).all()}
|
||||
assert synced_after == {f"vid{i}" for i in range(50, 251)}
|
||||
assert "vid49" not in synced_after
|
||||
|
||||
|
||||
def test_is_videos_sync_running_reflects_lock():
|
||||
sync._videos_lock.acquire()
|
||||
try:
|
||||
assert sync.is_videos_sync_running() is True
|
||||
finally:
|
||||
sync._videos_lock.release()
|
||||
assert sync.is_videos_sync_running() is False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue