diff --git a/bots/client_bot.py b/bots/client_bot.py index c18584d..845b714 100644 --- a/bots/client_bot.py +++ b/bots/client_bot.py @@ -262,9 +262,10 @@ async def my_players(update: Update, context: ContextTypes.DEFAULT_TYPE): for s in subs: safe_name = _md_escape(s['player_name']) safe_tour = _md_escape(s['tournament_name'] or '—') + last_pts_str = _md_escape(f'{s["last_points"]:.1f}') lines.append( f' {s["id"]}\\. {safe_name} \({s["rating"]}, {s["fed"]}\) ' - f'— {safe_tour} — {s["last_points"]:.1f} очк\\.') + f'— {safe_tour} — {last_pts_str} очк\\.') lines.append('\n/removeplayer <номер> — удалить подписку') await update.message.reply_text('\n'.join(lines), parse_mode=ParseMode.MARKDOWN_V2) @@ -315,19 +316,47 @@ async def _lookup_and_confirm(update: Update, context, fide_id: int): safe_name = _md_escape(player['name']) title_str = f', {player["title"]}' if player.get('title') else '' - - msg = ( + await update.message.reply_text( f'✅ *{safe_name}*{title_str}\n' f'{"🇫 " if player["fed"] else ""}{player["fed"]} \\| Рейтинг: {player["rating"]}\n\n' - 'Теперь отправь ссылку на турнир chess\\-results\\.com, ' - 'где играет этот игрок\.\n' - '/cancel — отмена' - ) - context.user_data['pending_fide'] = { - 'player': player, - 'lang': lang_code, - } - await update.message.reply_text(msg, parse_mode=ParseMode.MARKDOWN_V2) + '🔍 Ищу турниры на chess\\-results\\.com\\.\\.', + parse_mode=ParseMode.MARKDOWN_V2) + + # Run scan asynchronously (avoid blocking the event loop) + # Use canonical FIDE ID from profile page (may differ from URL-extracted ID) + from concurrent.futures import ThreadPoolExecutor + loop = asyncio.get_event_loop() + tournaments = await loop.run_in_executor( + None, tracker.scan_for_player, player['fide_id']) + + if tournaments: + # Auto-subscribe to all found tournaments + lines = [f'✅ *Найдено турниров: {len(tournaments)}*'] + for t in tournaments: + tracker.add_subscription( + user.id, player, t['url'], t['name'], t['sno'], + 0, 0.0, lang_code, t['start_date'], t['end_date']) + dates = '' + if t['start_date'] and t['end_date']: + dates = f' \({t["start_date"]} — {t["end_date"]}\)' + safe_tour = _md_escape(t['name']) + lines.append(f' • {safe_tour}{dates}') + lines.append('\nЯ буду присылать результаты и жеребьёвку\.\n' + '/myplayers — список подписок') + await update.message.reply_text('\n'.join(lines), + parse_mode=ParseMode.MARKDOWN_V2) + else: + # No tournaments found — fallback to manual URL + context.user_data['pending_fide'] = { + 'player': player, + 'lang': lang_code, + } + await update.message.reply_text( + '❌ Турниров не найдено автоматически\\.\n\n' + 'Отправь ссылку на турнир chess\\-results\\.com, ' + 'где играет этот игрок\\.\n' + '/cancel — отмена', + parse_mode=ParseMode.MARKDOWN_V2) def _apply_round_results(tournament, games, rd): @@ -442,7 +471,7 @@ async def _subscribe_to_tournament(update: Update, context, url: str, pending: d await msg.edit_text( f'✅ *{safe_name}* найден в турнире\\.\n' f'📅 *{safe_tour}*\n' - f'Ст\\.№ {p_sno}, {points:.1f} очк\\. — отслеживаю\!\n\n' + f'Ст\\.№ {p_sno}, {_md_escape(f"{points:.1f}")} очк\\. — отслеживаю\!\n\n' 'Я буду присылать результаты игрока и жеребьёвку следующих туров\.\n' '/myplayers — список подписок', parse_mode=ParseMode.MARKDOWN_V2) @@ -630,6 +659,12 @@ def main(): app.job_queue.run_repeating( tracker.check_all_subscriptions, interval=300, first=10) + app.job_queue.run_repeating( + tracker.rescan_new_tournaments, interval=3600, first=60) + app.job_queue.run_repeating( + tracker.warmup_cache, interval=21600, first=10) + + print('Client bot started', file=sys.stderr) print('Client bot started', file=sys.stderr) app.run_polling() diff --git a/bots/tracker.py b/bots/tracker.py index 477fb78..2a33fdb 100644 --- a/bots/tracker.py +++ b/bots/tracker.py @@ -2,6 +2,7 @@ Player tracking: FIDE profile lookup, subscription management, background polling for new results and pairings. """ +import json import re import os import sys @@ -32,6 +33,12 @@ RESULT_LOCALE = { } +def _migrate_add_column(conn, table: str, column: str, col_def: str): + cols = [r[1] for r in conn.execute(f'PRAGMA table_info({table})').fetchall()] + if column not in cols: + conn.execute(f'ALTER TABLE {table} ADD COLUMN {column} {col_def}') + + def _get_conn(): p = Path(DB_PATH) try: @@ -57,10 +64,31 @@ def _get_conn(): last_results_count INTEGER DEFAULT 0, last_points REAL DEFAULT 0.0, last_round_done INTEGER DEFAULT 0, + start_date TEXT DEFAULT '', + end_date TEXT DEFAULT '', active INTEGER DEFAULT 1, UNIQUE(user_id, fide_id, tournament_url) ) ''') + conn.execute(''' + CREATE TABLE IF NOT EXISTS tnr_state ( + key TEXT PRIMARY KEY, + value TEXT + ) + ''') + conn.execute(''' + CREATE TABLE IF NOT EXISTS tnr_cache ( + tnr INTEGER PRIMARY KEY, + name TEXT DEFAULT '', + start_date TEXT DEFAULT '', + end_date TEXT DEFAULT '', + players_json TEXT DEFAULT '{}', + scanned_at TEXT DEFAULT (datetime('now')) + ) + ''') + # Migrate existing tables that may be missing columns added in later versions + _migrate_add_column(conn, 'subscriptions', 'start_date', "TEXT DEFAULT ''") + _migrate_add_column(conn, 'subscriptions', 'end_date', "TEXT DEFAULT ''") conn.commit() return conn @@ -107,6 +135,11 @@ def fetch_fide_player(fide_id: int) -> dict: # Some players have multiple titles in adjacent

— take only the first one title = re.sub(r'<[^>]+>', '', title).strip() + # Canonical FIDE ID from profile page:

1503014

+ m = re.search(r'\s*(\d+)\s*

', html) + if m: + fide_id = int(m.group(1)) + return { 'fide_id': fide_id, 'name': name, 'rating': rating, 'fed': fed, 'title': title, @@ -118,17 +151,17 @@ def fetch_fide_player(fide_id: int) -> dict: def add_subscription(user_id: int, player: dict, tournament_url: str, tournament_name: str, player_sno: int, last_results_count: int, last_points: float, - lang: str): + lang: str, start_date: str = '', end_date: str = ''): conn = _get_conn() conn.execute(''' INSERT OR REPLACE INTO subscriptions (user_id, fide_id, player_name, rating, fed, lang, tournament_url, tournament_name, player_sno, last_results_count, last_points, - last_round_done, active) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 1) + last_round_done, start_date, end_date, active) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, 1) ''', (user_id, player['fide_id'], player['name'], player['rating'], player.get('fed', ''), lang, tournament_url, tournament_name, - player_sno, last_results_count, last_points)) + player_sno, last_results_count, last_points, start_date, end_date)) conn.commit() conn.close() @@ -264,6 +297,7 @@ def format_result(player_name: str, result: dict, opponent_name: str, safe_name = _md_escape_tracker(player_name) safe_opp = _md_escape_tracker(opponent_name) + safe_points = _md_escape_tracker(f'{points:.1f}') return ( f'♟ *{safe_name}* — {loc["round"]} {rd}\n' @@ -272,7 +306,385 @@ def format_result(player_name: str, result: dict, opponent_name: str, ) -# ═══ Background check ═══ +# ═══ TNR state cache ═══ + +DEFAULT_MAX_TNR = 500000 + + +def get_tnr_state(key: str, default: str = '') -> str: + conn = _get_conn() + row = conn.execute( + 'SELECT value FROM tnr_state WHERE key = ?', (key,)).fetchone() + conn.close() + return row[0] if row else default + + +def set_tnr_state(key: str, value: str): + conn = _get_conn() + conn.execute( + 'INSERT OR REPLACE INTO tnr_state (key, value) VALUES (?, ?)', + (key, value)) + conn.commit() + conn.close() + + +# ═══ TNR cache (avoids re-fetching art=0 pages) ═══ + +def _get_cached_tnr(tnr: int) -> dict | None: + conn = _get_conn() + row = conn.execute( + 'SELECT name, start_date, end_date, players_json FROM tnr_cache WHERE tnr = ?', + (tnr,)).fetchone() + conn.close() + if row is None: + return None + players_raw = json.loads(row[3]) + players = {int(k): v for k, v in players_raw.items()} + return { + 'name': row[0], 'start_date': row[1], 'end_date': row[2], + 'players': players, + } + + +def _save_tnr_cache(tnr: int, data: dict): + players_raw = {str(k): v for k, v in data.get('players', {}).items()} + conn = _get_conn() + conn.execute( + 'INSERT OR REPLACE INTO tnr_cache (tnr, name, start_date, end_date, players_json) ' + 'VALUES (?, ?, ?, ?, ?)', + (tnr, data.get('name', ''), data.get('start_date', ''), + data.get('end_date', ''), json.dumps(players_raw, ensure_ascii=False))) + conn.commit() + conn.close() + + +def _fetch_and_parse_art0(tnr: int) -> dict | None: + """Fetch art=0 page (or use cache), return parsed dict or None.""" + cached = _get_cached_tnr(tnr) + if cached is not None: + return cached + html = _fetch_art0(tnr) + if html is None: + return None + parsed = _parse_art0_page(html) + _save_tnr_cache(tnr, parsed) + return parsed + + +# ═══ Tournament scanner (art=0 pages) ═══ + +ART0_URL = 'https://chess-results.com/tnr{tnr}.aspx?lan=11&art=0&turdet=YES' + + +def _parse_art0_page(html: str) -> dict: + """Parse art=0 page: extract tournament name, dates, and list of players.""" + from bs4 import BeautifulSoup + + result = { + 'name': '', + 'start_date': '', + 'end_date': '', + 'players': {}, # fide_id -> {sno, name, rating, fed} + } + + soup = BeautifulSoup(html, 'html.parser') + + # Tournament name: from

+ h2 = soup.find('h2') + if h2: + result['name'] = h2.get_text(strip=True) + + # Dates: look for table row with "Дата(ы)" label + for tr in soup.find_all('tr'): + tds = tr.find_all('td') + texts = [td.get_text(strip=True) for td in tds] + for i, t in enumerate(texts): + if 'Дата' in t and i + 1 < len(texts): + m = re.match( + r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})', + texts[i + 1]) + if m: + result['start_date'] = m.group(1).replace('/', '-') + result['end_date'] = m.group(2).replace('/', '-') + break + if 'Date' in t and '(' in t and i + 1 < len(texts): + m = re.match( + r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})', + texts[i + 1]) + if m: + result['start_date'] = m.group(1).replace('/', '-') + result['end_date'] = m.group(2).replace('/', '-') + break + + # Players: find the main table with SNo/Name/FIDE/FED/Rating columns + for table in soup.find_all('table'): + rows = table.find_all('tr') + player_rows = 0 + for row in rows: + cells = row.find_all('td') + texts = [c.get_text(strip=True) for c in cells] + if len(texts) >= 4 and texts[0].isdigit() and re.search(r'[A-Z]{3}', ' '.join(texts)): + player_rows += 1 + if player_rows < 5: + continue + + for row in rows: + cells = row.find_all('td') + texts = [c.get_text(strip=True) for c in cells] + if len(texts) < 4 or not texts[0].isdigit(): + continue + + try: + sno = int(texts[0]) + except ValueError: + continue + + # Find FIDE ID column: numeric, 6-10 digits, near a FED column + fide_id = 0 + fed = '' + rating = 0 + name = '' + for ci, t in enumerate(texts): + if re.match(r'^\d{6,10}$', t): + fide_id = int(t) + if ci + 1 < len(texts) and re.match(r'^[A-Z]{3}$', texts[ci + 1]): + fed = texts[ci + 1] + if ci + (2 if fed else 1) < len(texts): + rt = texts[ci + (2 if fed else 1)] + if rt.isdigit(): + rating = int(rt) + if ci >= 1 and re.search(r'[A-Za-zА-Яа-я]', t) and len(t) > 2: + if not name: + name = t + + if fide_id: + result['players'][fide_id] = { + 'sno': sno, 'name': name, 'rating': rating, 'fed': fed, + } + + if result['players']: + break + + return result + + +def _fetch_art0(tnr: int) -> str | None: + """Fetch art=0 page, return HTML or None if not a valid tournament.""" + import time + url = ART0_URL.format(tnr=tnr) + for attempt in range(3): + try: + resp = requests.get(url, headers=HEADERS, timeout=15) + if resp.status_code == 429: + time.sleep(2 * (attempt + 1)) + continue + if resp.status_code != 200: + return None + resp.encoding = 'utf-8' + html = resp.text + # chess-results returns 200 even for non-existent tnrs + # Valid tournaments have

and are >30KB + if ' int: + """Find the current maximum tnr by probing upwards from saved value.""" + import time + saved = get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR)) + tnr = int(saved) + + for step in [10000, 1000, 100, 50, 20, 10, 5, 1]: + while True: + parsed = _fetch_and_parse_art0(tnr + step) + if parsed is not None: + tnr += step + else: + break + time.sleep(0.3) + + if tnr > int(saved): + set_tnr_state('max_tnr_seen', str(tnr)) + + return tnr + + +def scan_for_player(fide_id: int, max_tnr: int = None, + limit: int = 200) -> list: + """Scan tournaments for a player by FIDE ID. + + Scans multiple TNR windows since chess-results numbering has sparse clusters. + Returns list of {tnr, url, name, start_date, end_date, sno, player_name, fide_id}. + """ + import time + + results = [] + seen_tnrs = set() + + # Real TNR range on chess-results is not contiguous — multiple clusters exist. + # Scan from multiple starting points to cover both current (\u223c1.44M) and + # older (\u223c500K) clusters, plus a forward probe from the saved max. + saved = int(get_tnr_state('max_tnr_seen', '0')) + if max_tnr is None: + max_tnr = max(saved, discover_max_tnr()) + + # Scan windows: (start, limit) — cover known clusters + windows = [(max_tnr, limit)] + # Search tnr_cache for the player in the 1.4M cluster (instant SQL, no HTTP). + # Cache is populated by warmup_cache at startup (step 5 full scan). + for cluster_lo, cluster_hi in [(1434000, 1450000)]: + conn = _get_conn() + cached_tnrs = [ + r[0] for r in conn.execute( + 'SELECT tnr FROM tnr_cache WHERE tnr >= ? AND tnr <= ? ORDER BY tnr DESC', + (cluster_lo, cluster_hi)).fetchall() + ] + conn.close() + for tnr in cached_tnrs: + if tnr in seen_tnrs: + continue + seen_tnrs.add(tnr) + cdata = _get_cached_tnr(tnr) + if cdata is None: + continue + player_info = cdata['players'].get(fide_id) + if player_info is None: + continue + results.append({ + 'tnr': tnr, + 'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11', + 'name': cdata['name'] or f'Tournament {tnr}', + 'start_date': cdata['start_date'], + 'end_date': cdata['end_date'], + 'sno': player_info['sno'], + 'player_name': player_info['name'], + 'fide_id': fide_id, + }) + if results: + set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results))) + return results + + # Fallback: sequential scan from discovered max_tnr window + for start, win_limit in windows: + for tnr in range(start, max(1, start - win_limit), -1): + if tnr in seen_tnrs: + continue + seen_tnrs.add(tnr) + parsed = _fetch_and_parse_art0(tnr) + if parsed is None: + continue + if not parsed['players']: + continue + player_info = parsed['players'].get(fide_id) + if player_info is None: + continue + results.append({ + 'tnr': tnr, + 'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11', + 'name': parsed['name'] or f'Tournament {tnr}', + 'start_date': parsed['start_date'], + 'end_date': parsed['end_date'], + 'sno': player_info['sno'], + 'player_name': player_info['name'], + 'fide_id': fide_id, + }) + time.sleep(0.3) + + if results: + set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results))) + elif max_tnr > saved: + set_tnr_state('max_tnr_seen', str(max_tnr)) + + return results + + +def get_unique_fide_ids() -> list: + """Return unique FIDE IDs from active subscriptions.""" + conn = _get_conn() + rows = conn.execute( + 'SELECT DISTINCT fide_id FROM subscriptions WHERE active = 1').fetchall() + conn.close() + return [r[0] for r in rows] + + +def check_new_tournaments_for_player(fide_id: int, new_max_tnr: int, + prev_max_tnr: int) -> list: + """Check new tnr range for a specific player. Returns new tournaments found.""" + results = [] + for tnr in range(new_max_tnr, prev_max_tnr, -1): + parsed = _fetch_and_parse_art0(tnr) + if parsed is None: + continue + if not parsed['players']: + continue + player_info = parsed['players'].get(fide_id) + if player_info is None: + continue + url = f'https://chess-results.com/tnr{tnr}.aspx?lan=11' + results.append({ + 'tnr': tnr, 'url': url, + 'name': parsed['name'] or f'Tournament {tnr}', + 'start_date': parsed['start_date'], + 'end_date': parsed['end_date'], + 'sno': player_info['sno'], + 'player_name': player_info['name'], + 'fide_id': fide_id, + }) + return results + + +async def warmup_cache(context): + """Build full tnr_cache index (runs once at startup, then every 6h).""" + import asyncio, sys, traceback + + # Skip if warmup already completed + status = get_tnr_state('warmup_tnr', '') + if status == 'done': + return + + print('warmup_cache: starting cluster scan (step=1, 1445000→1434000)...', file=sys.stderr) + try: + await asyncio.to_thread(_warmup_cache_sync) + print('warmup_cache: cluster scan complete', file=sys.stderr) + except Exception as e: + print(f'warmup_cache: FAILED: {e}', file=sys.stderr) + traceback.print_exc(file=sys.stderr) + + +def _warmup_cache_sync(): + """Scan the 1.4M cluster at step 1, caching ALL valid TNRs with players. + One-time cost ~40 min, then all searches are instant SQL queries. + Saves progress to tnr_state so it resumes after restart.""" + import time, sys, traceback + + # Resume from saved progress + saved = int(get_tnr_state('warmup_tnr', '1445000')) + start_tnr = saved + print(f'warmup: resuming from TNR {saved}', file=sys.stderr) + + for tnr in range(start_tnr, 1433999, -1): + try: + _fetch_and_parse_art0(tnr) + except Exception as e: + print(f'warmup: error at TNR {tnr}: {e}', file=sys.stderr) + traceback.print_exc(file=sys.stderr) + time.sleep(2) + continue + + if tnr % 100 == 0: + set_tnr_state('warmup_tnr', str(tnr)) + if tnr % 500 == 0: + elapsed = (start_tnr - tnr) // 500 * 500 # rough + print(f'warmup: progress TNR {tnr} (started at {start_tnr})', file=sys.stderr) + time.sleep(0.02) + + # Mark warmup as complete — next run will see this and skip + set_tnr_state('warmup_tnr', 'done') + async def check_all_subscriptions(context): from swiss_calc.parser import fetch_tournament @@ -329,3 +741,52 @@ async def check_all_subscriptions(context): except Exception: pass update_round(sub['id'], current_rd) + + +async def rescan_new_tournaments(context): + """Hourly: discover new tnrs and check if subscribed players appear.""" + prev_max = int(get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR))) + new_max = discover_max_tnr() + + if new_max <= prev_max: + return + + fide_ids = get_unique_fide_ids() + if not fide_ids: + return + + conn = _get_conn() + user_fide_map = {} + rows = conn.execute( + 'SELECT DISTINCT user_id, fide_id, lang FROM subscriptions WHERE active = 1' + ).fetchall() + conn.close() + for r in rows: + user_fide_map.setdefault(r[1], []).append((r[0], r[2])) + + for fide_id in fide_ids: + new_tournaments = check_new_tournaments_for_player( + fide_id, new_max, prev_max) + for t in new_tournaments: + for user_id, lang in user_fide_map.get(fide_id, []): + player = {'fide_id': fide_id, 'name': t['player_name'], + 'rating': 0, 'fed': ''} + add_subscription( + user_id, player, t['url'], t['name'], t['sno'], + 0, 0.0, lang, t['start_date'], t['end_date']) + try: + safe_name = _md_escape_tracker(t['player_name']) + safe_tour = _md_escape_tracker(t['name']) + dates = '' + if t['start_date'] and t['end_date']: + dates = f'\\({t["start_date"]} — {t["end_date"]}\\)' + await context.bot.send_message( + user_id, + f'🆕 *{safe_name}* найден в новом турнире\!\n' + f'📅 *{safe_tour}*{dates}\n' + 'Автоматически подписал — буду отслеживать\.', + parse_mode=ParseMode.MARKDOWN_V2) + except Exception: + pass + + set_tnr_state('max_tnr_seen', str(new_max))