fix periodic activity-check algorithm silently losing player activity
All checks were successful
CI/CD Pipeline / build-and-deploy (push) Successful in 38s
All checks were successful
CI/CD Pipeline / build-and-deploy (push) Successful in 38s
Errors from Lichess (timeouts/5xx/invalid tokens) were being disguised as "no activity" (HTTP 200, games_count/puzzles_in_period=0), causing the bot to silently advance its checkpoint past real, undetected activity. Puzzle-fetch failures weren't counted as errors at all, and the periodic task died permanently after 5 consecutive errors with no way to recover short of a manual restart. /setperiod also unconditionally reset the checkpoint, dropping the window between the last check and the command. - API now returns success=false/502 on real errors instead of masking them as zero activity (models.py, stats_service.py, main.py) - Puzzle-fetch errors are now treated the same as game-fetch errors: retry the same window instead of reporting "no puzzles" - Notification delivery failures no longer silently advance the checkpoint - Replaced the hard 5-error kill switch with capped backoff that keeps retrying indefinitely, plus an admin-bot notification if a player's monitoring has been failing for a prolonged period (~2h+) - /setperiod only clears the checkpoint when disabling notifications, preserving continuity when a period is just changed Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
vrubelroman
parent
75e62b57a1
commit
4a783225af
4 changed files with 118 additions and 58 deletions
|
|
@ -134,7 +134,45 @@ class LichessBot:
|
|||
logger.error(f"Failed to send admin notification via API: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _estimate_backoff_elapsed_hours(consecutive_errors: int) -> float:
|
||||
"""Оценивает, сколько времени прошло при данном числе подряд идущих ошибок,
|
||||
исходя из формулы бэкоффа periodic_check: min(300, 60 * n) секунд на попытку."""
|
||||
total_seconds = sum(min(300, 60 * i) for i in range(1, consecutive_errors + 1))
|
||||
return total_seconds / 3600
|
||||
|
||||
async def _notify_admin_periodic_check_failing(self, gamer_username: str, user_id: int, consecutive_errors: int):
|
||||
"""Notify admin that periodic check for a gamer has been failing for a long time."""
|
||||
try:
|
||||
admin_chat_id = self.db.get_admin_chat_id()
|
||||
if not admin_chat_id:
|
||||
logger.warning("Admin chat id is not set; cannot send admin notification.")
|
||||
return
|
||||
hours = self._estimate_backoff_elapsed_hours(consecutive_errors)
|
||||
url = f"https://api.telegram.org/bot{ADMINPANEL_TELEGRAM_BOT_TOKEN}/sendMessage"
|
||||
message = (
|
||||
f"⚠️ <b>Мониторинг игрока не работает</b>\n\n"
|
||||
f"Игрок: {gamer_username}\n"
|
||||
f"Владелец: user_id={user_id}\n"
|
||||
f"Подряд идущих ошибок: {consecutive_errors} (~{hours:.1f} ч)\n\n"
|
||||
f"Проверьте токен игрока или доступность Lichess API."
|
||||
)
|
||||
logger.info(f"Sending admin 'periodic check failing' notification via direct API to chat_id={admin_chat_id}")
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(url, json={
|
||||
"chat_id": admin_chat_id,
|
||||
"text": message,
|
||||
"parse_mode": "HTML"
|
||||
}) as response:
|
||||
if response.status == 200:
|
||||
logger.info("Admin 'periodic check failing' notification sent successfully")
|
||||
else:
|
||||
error_text = await response.text()
|
||||
logger.error(f"Failed to send admin 'periodic check failing' notification: {response.status} - {error_text}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send admin 'periodic check failing' notification: {e}")
|
||||
|
||||
async def test_admin_notify(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||
"""Manual test to verify admin notifications delivery path."""
|
||||
user = update.effective_user
|
||||
|
|
@ -1284,9 +1322,11 @@ class LichessBot:
|
|||
|
||||
# Set period for this user-gamer pair
|
||||
self.db.set_user_gamer_period(user_id, gamer_id, period)
|
||||
self.db.clear_period_checkpoint(user_id, gamer_id)
|
||||
|
||||
|
||||
if period == 0:
|
||||
# Отключаем мониторинг: чистим чекпоинт, чтобы при повторном включении
|
||||
# не было catch-up на потенциально очень старое (устаревшее) окно.
|
||||
self.db.clear_period_checkpoint(user_id, gamer_id)
|
||||
await self.stop_periodic_task(gamer_id, user_id)
|
||||
await query.edit_message_text(
|
||||
t('notifications_disabled', lang, username=selected_gamer['username'])
|
||||
|
|
@ -1446,6 +1486,10 @@ class LichessBot:
|
|||
)
|
||||
self.periodic_tasks[task_key] = task
|
||||
|
||||
# Оповещать админ-бота о затяжной серии ошибок каждые ADMIN_NOTIFY_ERROR_THRESHOLD
|
||||
# подряд идущих ошибок (~2 часа при капнутом бэкоффе в 300с на попытку)
|
||||
ADMIN_NOTIFY_ERROR_THRESHOLD = 25
|
||||
|
||||
async def periodic_check(self, gamer: Dict[str, Any], user_id: int, period_minutes: int):
|
||||
"""Periodic check for gamer activity"""
|
||||
task_key = f"{gamer['id']}_{user_id}"
|
||||
|
|
@ -1551,27 +1595,33 @@ class LichessBot:
|
|||
except Exception as e:
|
||||
logger.error(f"❌ Error getting games data for {gamer['username']}: {e}")
|
||||
consecutive_errors += 1
|
||||
backoff_seconds = min(300, 60 * consecutive_errors)
|
||||
if consecutive_errors >= max_consecutive_errors:
|
||||
logger.error(f"Too many consecutive errors for {gamer['username']}, stopping periodic check")
|
||||
break
|
||||
logger.warning(f"⚠️ Games data unavailable for {gamer['username']}; retrying the same period in 60 seconds")
|
||||
await asyncio.sleep(60)
|
||||
logger.error(f"⚠️ {consecutive_errors} consecutive errors for {gamer['username']}; still retrying the same period (backoff {backoff_seconds}s)")
|
||||
else:
|
||||
logger.warning(f"⚠️ Games data unavailable for {gamer['username']}; retrying the same period in {backoff_seconds}s")
|
||||
if consecutive_errors == self.ADMIN_NOTIFY_ERROR_THRESHOLD or (
|
||||
consecutive_errors > self.ADMIN_NOTIFY_ERROR_THRESHOLD
|
||||
and consecutive_errors % self.ADMIN_NOTIFY_ERROR_THRESHOLD == 0
|
||||
):
|
||||
await self._notify_admin_periodic_check_failing(gamer['username'], user_id, consecutive_errors)
|
||||
await asyncio.sleep(backoff_seconds)
|
||||
continue
|
||||
|
||||
|
||||
if gamer.get('token'):
|
||||
try:
|
||||
# Добавляем запрос в очередь (будет выполнен с задержкой 7 секунд)
|
||||
logger.info(f"📥 Adding puzzles request to queue for {gamer['username']}")
|
||||
puzzles_data = await self.request_queue.add_request(
|
||||
self.lichess_api.get_puzzles_period,
|
||||
gamer['token'], since_timestamp, until_timestamp_approx, 150
|
||||
)
|
||||
# Обновляем фактическое время после получения ответа по пазлам
|
||||
request_end_time = datetime.now()
|
||||
logger.info(f"✅ Puzzles API response received for {gamer['username']} at {request_end_time}")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Error getting puzzles data for {gamer['username']}: {e}")
|
||||
# Продолжаем без данных по пазлам
|
||||
# Ошибки получения пазлов обрабатываются так же, как ошибки игр:
|
||||
# исключение всплывает во внешний обработчик, чекпоинт не продвигается,
|
||||
# то же окно ретраится на следующей итерации (не считаем "пазлов не было").
|
||||
logger.info(f"📥 Adding puzzles request to queue for {gamer['username']}")
|
||||
puzzles_data = await self.request_queue.add_request(
|
||||
self.lichess_api.get_puzzles_period,
|
||||
gamer['token'], since_timestamp, until_timestamp_approx, 150
|
||||
)
|
||||
if puzzles_data is None:
|
||||
raise RuntimeError("Puzzles period API returned no data")
|
||||
# Обновляем фактическое время после получения ответа по пазлам
|
||||
request_end_time = datetime.now()
|
||||
logger.info(f"✅ Puzzles API response received for {gamer['username']} at {request_end_time}")
|
||||
|
||||
# Сбрасываем счетчик ошибок при успешном запросе
|
||||
consecutive_errors = 0
|
||||
|
|
@ -1628,35 +1678,28 @@ class LichessBot:
|
|||
|
||||
logger.info(f"🔍 Activity check result for {username}: has_games={has_games} (total={total_games}), has_puzzles={has_puzzles} (total={total_puzzles})")
|
||||
|
||||
# Отправляем уведомление только если есть реальная активность
|
||||
# Отправляем уведомление только если есть реальная активность.
|
||||
# Ошибки форматирования/отправки НЕ перехватываются здесь: они должны
|
||||
# всплыть во внешний обработчик, чтобы чекпоинт не продвинулся и уже
|
||||
# обнаруженная активность была отправлена повторно на следующей попытке,
|
||||
# а не потеряна молча.
|
||||
if has_games or has_puzzles:
|
||||
logger.info(f"📊 Activity detected for {gamer['username']}, preparing notification...")
|
||||
try:
|
||||
# Get user language from database
|
||||
user_lang = self.db.get_user_language(user_id)
|
||||
notification = StatsFormatter.format_period_notification(
|
||||
gamer['username'], games_data, puzzles_data, period_minutes, lang=user_lang
|
||||
)
|
||||
|
||||
if self.application:
|
||||
try:
|
||||
await self.application.bot.send_message(
|
||||
chat_id=user_id,
|
||||
text=notification
|
||||
)
|
||||
logger.info(f"✅ Sent periodic notification for {gamer['username']} to user {user_id}")
|
||||
# Increment periodic notification counter
|
||||
self.counters.increment('periodic_notification')
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Failed to send notification to user {user_id}: {e}")
|
||||
import traceback
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
else:
|
||||
logger.error(f"❌ Application not initialized, cannot send notification for {gamer['username']} to user {user_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error formatting notification for {gamer['username']}: {e}")
|
||||
import traceback
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
user_lang = self.db.get_user_language(user_id)
|
||||
notification = StatsFormatter.format_period_notification(
|
||||
gamer['username'], games_data, puzzles_data, period_minutes, lang=user_lang
|
||||
)
|
||||
|
||||
if not self.application:
|
||||
raise RuntimeError(f"Application not initialized, cannot send notification for {gamer['username']} to user {user_id}")
|
||||
|
||||
await self.application.bot.send_message(
|
||||
chat_id=user_id,
|
||||
text=notification
|
||||
)
|
||||
logger.info(f"✅ Sent periodic notification for {gamer['username']} to user {user_id}")
|
||||
# Increment periodic notification counter
|
||||
self.counters.increment('periodic_notification')
|
||||
else:
|
||||
logger.debug(f"⏭️ No activity found for {gamer['username']} in the last {period_minutes} minutes")
|
||||
|
||||
|
|
@ -1686,17 +1729,19 @@ class LichessBot:
|
|||
logger.error(f"Error in periodic check for {gamer['username']}: {e}")
|
||||
import traceback
|
||||
logger.error(f"Full traceback: {traceback.format_exc()}")
|
||||
|
||||
if consecutive_errors >= max_consecutive_errors:
|
||||
logger.error(f"Too many consecutive errors for {gamer['username']}, stopping periodic check")
|
||||
if task_key in self.periodic_tasks:
|
||||
del self.periodic_tasks[task_key]
|
||||
if task_key in self.period_start_times:
|
||||
del self.period_start_times[task_key]
|
||||
break
|
||||
|
||||
# Ждем перед повторной попыткой при ошибке
|
||||
await asyncio.sleep(60) # 1 minute delay before retry
|
||||
# Не убиваем задачу навсегда: ретраим то же окно с растущим бэкоффом
|
||||
# (капается на 300с), пока Lichess/Telegram не восстановятся.
|
||||
backoff_seconds = min(300, 60 * consecutive_errors)
|
||||
if consecutive_errors >= max_consecutive_errors:
|
||||
logger.error(f"⚠️ {consecutive_errors} consecutive errors for {gamer['username']}; still retrying the same period (backoff {backoff_seconds}s)")
|
||||
if consecutive_errors == self.ADMIN_NOTIFY_ERROR_THRESHOLD or (
|
||||
consecutive_errors > self.ADMIN_NOTIFY_ERROR_THRESHOLD
|
||||
and consecutive_errors % self.ADMIN_NOTIFY_ERROR_THRESHOLD == 0
|
||||
):
|
||||
await self._notify_admin_periodic_check_failing(gamer['username'], user_id, consecutive_errors)
|
||||
await asyncio.sleep(backoff_seconds)
|
||||
continue
|
||||
|
||||
def setup_handlers(self, application: Application):
|
||||
"""Setup all handlers"""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue