Channel features: subscribers, new-videos badges, activity-driven sync
- Store subscriber count from YouTube statistics and show it on channel page - Sync 50 videos per channel with playlistItems pagination support - Show per-channel and per-category new-videos counters (2-day window) - Replace hourly videos sync with activity trigger (2h idle) and incremental backfill (hard cap 200 per channel) - Clicking the sidebar new-videos count filters the category feed to recent videos only (new_only) - Update agent-team docs: deploy after green checks
This commit is contained in:
vrubelroman
parent
fde9a439df
commit
e10df8dcbd
27 changed files with 1420 additions and 107 deletions
|
|
@ -52,6 +52,10 @@ def get_videos_sync_status(db: Session) -> dict:
|
|||
return _get_status(db, VIDEOS_SYNC_STATUS_KEY, _videos_lock)
|
||||
|
||||
|
||||
def is_videos_sync_running() -> bool:
|
||||
return _videos_lock.locked()
|
||||
|
||||
|
||||
def sync_subscriptions(db: Session) -> dict:
|
||||
if not _subscriptions_lock.acquire(blocking=False):
|
||||
raise SyncInProgress("Subscriptions sync already in progress")
|
||||
|
|
@ -108,10 +112,13 @@ def sync_subscriptions(db: Session) -> dict:
|
|||
subscribed_ids = [c.youtube_channel_id for c in existing.values() if c.subscribed]
|
||||
try:
|
||||
uploads = youtube_client.fetch_uploads_playlists(credentials, subscribed_ids)
|
||||
for channel_id, uploads_playlist_id in uploads.items():
|
||||
for channel_id, data in uploads.items():
|
||||
channel = existing.get(channel_id)
|
||||
if channel is not None:
|
||||
channel.uploads_playlist_id = uploads_playlist_id
|
||||
if data["uploads_playlist_id"] is not None:
|
||||
channel.uploads_playlist_id = data["uploads_playlist_id"]
|
||||
if data["subscriber_count"] is not None:
|
||||
channel.subscriber_count = data["subscriber_count"]
|
||||
db.commit()
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch uploads playlists during subscriptions sync")
|
||||
|
|
@ -166,21 +173,38 @@ def sync_videos(db: Session) -> dict:
|
|||
)
|
||||
channel_by_youtube_id = {c.youtube_channel_id: c for c in channels}
|
||||
|
||||
# One query for all known video ids (grouped by channel) so there is
|
||||
# no N+1 at the DB level; the YouTube API is still queried per channel.
|
||||
known_ids_by_channel: dict[int, set[str]] = {}
|
||||
for youtube_video_id, channel_id in db.query(Video.youtube_video_id, Video.channel_id).all():
|
||||
known_ids_by_channel.setdefault(channel_id, set()).add(youtube_video_id)
|
||||
|
||||
candidate_video_ids: set[str] = set()
|
||||
for channel in channels:
|
||||
try:
|
||||
video_ids = youtube_client.fetch_playlist_video_ids(
|
||||
credentials, channel.uploads_playlist_id, settings.videos_per_channel_sync
|
||||
# videos_backfill_cap is a hard history-depth limit per
|
||||
# channel: only the newest N playlist items are ever
|
||||
# considered; anything older is not backfilled by design.
|
||||
# The early stop on consecutive known ids saves playlistItems
|
||||
# pages on repeated syncs inside that window.
|
||||
new_ids = youtube_client.fetch_playlist_video_ids_incremental(
|
||||
credentials,
|
||||
channel.uploads_playlist_id,
|
||||
known_ids_by_channel.get(channel.id, set()),
|
||||
settings.videos_backfill_cap,
|
||||
settings.videos_known_stop_threshold,
|
||||
)
|
||||
candidate_video_ids.update(video_ids)
|
||||
candidate_video_ids.update(new_ids)
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch playlist items for channel %s", channel.youtube_channel_id)
|
||||
|
||||
# Details are fetched only for ids that are not in the DB yet (the
|
||||
# incremental fetch above already filters out known ids), so every
|
||||
# returned item is a new video to insert. Existing videos' metadata is
|
||||
# deliberately not refreshed by the videos sync.
|
||||
details = youtube_client.fetch_videos_details(credentials, list(candidate_video_ids))
|
||||
|
||||
existing = {v.youtube_video_id: v for v in db.query(Video).all()}
|
||||
added = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
|
||||
for item in details:
|
||||
|
|
@ -197,9 +221,8 @@ def sync_videos(db: Session) -> dict:
|
|||
duration_seconds = parse_iso8601_duration(item["duration_iso8601"])
|
||||
youtube_url = settings.youtube_watch_url_template.format(video_id=item["youtube_video_id"])
|
||||
|
||||
video = existing.get(item["youtube_video_id"])
|
||||
if video is None:
|
||||
video = Video(
|
||||
db.add(
|
||||
Video(
|
||||
youtube_video_id=item["youtube_video_id"],
|
||||
channel_id=channel.id,
|
||||
title=item["title"],
|
||||
|
|
@ -209,18 +232,8 @@ def sync_videos(db: Session) -> dict:
|
|||
duration_seconds=duration_seconds,
|
||||
youtube_url=youtube_url,
|
||||
)
|
||||
db.add(video)
|
||||
existing[item["youtube_video_id"]] = video
|
||||
added += 1
|
||||
else:
|
||||
video.channel_id = channel.id
|
||||
video.title = item["title"]
|
||||
video.description = item["description"]
|
||||
video.thumbnail_url = item["thumbnail_url"]
|
||||
video.published_at = published_at
|
||||
video.duration_seconds = duration_seconds
|
||||
video.youtube_url = youtube_url
|
||||
updated += 1
|
||||
)
|
||||
added += 1
|
||||
|
||||
db.commit()
|
||||
|
||||
|
|
@ -230,12 +243,14 @@ def sync_videos(db: Session) -> dict:
|
|||
"finished_at": _now_iso(),
|
||||
"error": None,
|
||||
"videos_added": added,
|
||||
"videos_updated": updated,
|
||||
# Kept for API compatibility: always 0, because the videos sync
|
||||
# never refreshes metadata of existing videos.
|
||||
"videos_updated": 0,
|
||||
"videos_skipped": skipped,
|
||||
"channels_checked": len(channels),
|
||||
}
|
||||
_save_status(db, VIDEOS_SYNC_STATUS_KEY, result)
|
||||
logger.info("Videos sync completed: added=%d updated=%d skipped=%d", added, updated, skipped)
|
||||
logger.info("Videos sync completed: added=%d skipped=%d", added, skipped)
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue