Channel features: subscribers, new-videos badges, activity-driven sync
- Store subscriber count from YouTube statistics and show it on channel page - Sync 50 videos per channel with playlistItems pagination support - Show per-channel and per-category new-videos counters (2-day window) - Replace hourly videos sync with activity trigger (2h idle) and incremental backfill (hard cap 200 per channel) - Clicking the sidebar new-videos count filters the category feed to recent videos only (new_only) - Update agent-team docs: deploy after green checks
This commit is contained in:
vrubelroman
parent
fde9a439df
commit
e10df8dcbd
27 changed files with 1420 additions and 107 deletions
|
|
@ -115,6 +115,160 @@ def test_fetch_playlist_video_ids_returns_empty_on_404(monkeypatch):
|
|||
assert result == []
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_paginates(monkeypatch):
|
||||
page1 = FakeResponse(
|
||||
200,
|
||||
{
|
||||
"items": [{"contentDetails": {"videoId": f"vid{i}"}} for i in range(50)],
|
||||
"nextPageToken": "page2",
|
||||
},
|
||||
)
|
||||
page2 = FakeResponse(
|
||||
200,
|
||||
{"items": [{"contentDetails": {"videoId": f"vid{i}"}} for i in range(50, 80)]},
|
||||
)
|
||||
seen = []
|
||||
|
||||
class RecordingClient(FakeClient):
|
||||
def get(self, url, params=None, headers=None):
|
||||
seen.append({"url": url, "params": params})
|
||||
return self._responses.pop(0)
|
||||
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: RecordingClient([page1, page2]))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids(_credentials(), "UUplaylist", 80)
|
||||
|
||||
assert result == [f"vid{i}" for i in range(80)]
|
||||
assert len(seen) == 2
|
||||
assert seen[0]["params"]["maxResults"] == 50
|
||||
assert "pageToken" not in seen[0]["params"]
|
||||
assert seen[1]["params"]["maxResults"] == 30
|
||||
assert seen[1]["params"]["pageToken"] == "page2"
|
||||
assert seen[0]["params"]["part"] == "contentDetails"
|
||||
assert seen[1]["params"]["part"] == "contentDetails"
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_stops_at_safety_cap(monkeypatch):
|
||||
page = FakeResponse(
|
||||
200,
|
||||
{"items": [{"contentDetails": {"videoId": "vid"}}], "nextPageToken": "next"},
|
||||
)
|
||||
seen = []
|
||||
|
||||
class RecordingClient(FakeClient):
|
||||
def get(self, url, params=None, headers=None):
|
||||
seen.append(params)
|
||||
return self._responses.pop(0)
|
||||
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: RecordingClient([page] * 100))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids(_credentials(), "UUplaylist", 10000)
|
||||
|
||||
assert len(result) == youtube_client.MAX_PLAYLIST_PAGES
|
||||
assert len(seen) == youtube_client.MAX_PLAYLIST_PAGES
|
||||
assert "pageToken" not in seen[0]
|
||||
assert seen[1]["pageToken"] == "next"
|
||||
|
||||
|
||||
def _page_items(video_ids, next_token=None):
|
||||
payload = {"items": [{"contentDetails": {"videoId": vid}} for vid in video_ids]}
|
||||
if next_token:
|
||||
payload["nextPageToken"] = next_token
|
||||
return FakeResponse(200, payload)
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_incremental_returns_only_unknown(monkeypatch):
|
||||
page = _page_items(["new1", "known1", "new2", "known2"])
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: FakeClient([page]))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
_credentials(), "UUplaylist", known_ids={"known1", "known2"}, max_results=10, stop_threshold=50
|
||||
)
|
||||
|
||||
assert result == ["new1", "new2"]
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_incremental_stops_on_known_run(monkeypatch):
|
||||
page1 = _page_items([f"known{i}" for i in range(50)], next_token="page2")
|
||||
page2 = _page_items(["new1", "new2"], next_token="page3")
|
||||
seen = []
|
||||
|
||||
class RecordingClient(FakeClient):
|
||||
def get(self, url, params=None, headers=None):
|
||||
seen.append(params)
|
||||
return self._responses.pop(0)
|
||||
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: RecordingClient([page1, page2]))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
_credentials(), "UUplaylist", known_ids={f"known{i}" for i in range(50)}, max_results=10, stop_threshold=50
|
||||
)
|
||||
|
||||
assert result == []
|
||||
assert len(seen) == 1 # page2 never requested
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_incremental_known_run_crosses_pages(monkeypatch):
|
||||
page1 = _page_items(["new1"] + [f"known{i}" for i in range(49)], next_token="page2")
|
||||
page2 = _page_items([f"known{i}" for i in range(49, 60)], next_token="page3")
|
||||
seen = []
|
||||
|
||||
class RecordingClient(FakeClient):
|
||||
def get(self, url, params=None, headers=None):
|
||||
seen.append(params)
|
||||
return self._responses.pop(0)
|
||||
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: RecordingClient([page1, page2]))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
_credentials(), "UUplaylist", known_ids={f"known{i}" for i in range(60)}, max_results=10, stop_threshold=50
|
||||
)
|
||||
|
||||
assert result == ["new1"]
|
||||
assert len(seen) == 2 # stopped mid-page-2, page3 never requested
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_incremental_stops_at_cap(monkeypatch):
|
||||
pages = [_page_items([f"vid{page * 50 + i}" for i in range(50)], next_token=f"p{page + 2}") for page in range(6)]
|
||||
seen = []
|
||||
|
||||
class RecordingClient(FakeClient):
|
||||
def get(self, url, params=None, headers=None):
|
||||
seen.append(params)
|
||||
return self._responses.pop(0)
|
||||
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: RecordingClient(pages))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
_credentials(), "UUplaylist", known_ids=set(), max_results=200, stop_threshold=50
|
||||
)
|
||||
|
||||
assert len(result) == 200
|
||||
assert len(seen) == 4
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_incremental_empty_on_404(monkeypatch):
|
||||
response = FakeResponse(404, {"error": {"message": "playlist not found"}})
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: FakeClient([response]))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
_credentials(), "UUplaylist", known_ids=set(), max_results=10, stop_threshold=50
|
||||
)
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_fetch_playlist_video_ids_incremental_dedupes_new_ids(monkeypatch):
|
||||
page = _page_items(["new1", "new1", "known1"])
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: FakeClient([page]))
|
||||
|
||||
result = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
_credentials(), "UUplaylist", known_ids={"known1"}, max_results=10, stop_threshold=50
|
||||
)
|
||||
|
||||
assert result == ["new1"]
|
||||
|
||||
|
||||
def test_fetch_videos_details(monkeypatch):
|
||||
response = FakeResponse(
|
||||
200,
|
||||
|
|
@ -194,13 +348,48 @@ def test_fetch_uploads_playlists_batches(monkeypatch):
|
|||
200,
|
||||
{
|
||||
"items": [
|
||||
{"id": "chanA", "contentDetails": {"relatedPlaylists": {"uploads": "UUchanA"}}},
|
||||
{"id": "chanB", "contentDetails": {"relatedPlaylists": {"uploads": "UUchanB"}}},
|
||||
{
|
||||
"id": "chanA",
|
||||
"contentDetails": {"relatedPlaylists": {"uploads": "UUchanA"}},
|
||||
"statistics": {"subscriberCount": "12345"},
|
||||
},
|
||||
{
|
||||
"id": "chanB",
|
||||
"contentDetails": {"relatedPlaylists": {"uploads": "UUchanB"}},
|
||||
"statistics": {"hiddenSubscriberCount": True},
|
||||
},
|
||||
{"id": "chanC", "contentDetails": {"relatedPlaylists": {"uploads": "UUchanC"}}},
|
||||
{"id": "chanD", "contentDetails": {"relatedPlaylists": {}}, "statistics": {"subscriberCount": "42"}},
|
||||
]
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: FakeClient([response]))
|
||||
|
||||
result = youtube_client.fetch_uploads_playlists(_credentials(), ["chanA", "chanB"])
|
||||
result = youtube_client.fetch_uploads_playlists(_credentials(), ["chanA", "chanB", "chanC", "chanD"])
|
||||
|
||||
assert result == {"chanA": "UUchanA", "chanB": "UUchanB"}
|
||||
assert result == {
|
||||
"chanA": {"uploads_playlist_id": "UUchanA", "subscriber_count": 12345},
|
||||
"chanB": {"uploads_playlist_id": "UUchanB", "subscriber_count": None},
|
||||
"chanC": {"uploads_playlist_id": "UUchanC", "subscriber_count": None},
|
||||
"chanD": {"uploads_playlist_id": None, "subscriber_count": 42},
|
||||
}
|
||||
|
||||
|
||||
def test_fetch_uploads_playlists_subscriber_count_not_an_int_is_none(monkeypatch):
|
||||
response = FakeResponse(
|
||||
200,
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"id": "chanA",
|
||||
"contentDetails": {"relatedPlaylists": {"uploads": "UUchanA"}},
|
||||
"statistics": {"subscriberCount": "not-a-number"},
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(youtube_client.httpx, "Client", lambda timeout: FakeClient([response]))
|
||||
|
||||
result = youtube_client.fetch_uploads_playlists(_credentials(), ["chanA"])
|
||||
|
||||
assert result == {"chanA": {"uploads_playlist_id": "UUchanA", "subscriber_count": None}}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue