""" Player tracking: FIDE profile lookup, subscription management, background polling for new results and pairings. """ import json import re import os import sys import sqlite3 import requests from pathlib import Path from datetime import date sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) DB_PATH = os.environ.get('STATS_DB', '/app/data/tournaments.db') FIDE_URL = 'https://ratings.fide.com/profile/{fide_id}' HEADERS = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36' } RESULT_LOCALE = { 'ru': { 'win': 'победа', 'draw': 'ничья', 'loss': 'поражение', 'white': 'белыми', 'black': 'чёрными', 'points': 'Очков', 'round': 'тур', 'vs': 'против', }, 'en': { 'win': 'win', 'draw': 'draw', 'loss': 'loss', 'white': 'White', 'black': 'Black', 'points': 'Points', 'round': 'round', 'vs': 'vs', }, } def _migrate_add_column(conn, table: str, column: str, col_def: str): cols = [r[1] for r in conn.execute(f'PRAGMA table_info({table})').fetchall()] if column not in cols: conn.execute(f'ALTER TABLE {table} ADD COLUMN {column} {col_def}') def _get_conn(): p = Path(DB_PATH) try: p.parent.mkdir(parents=True, exist_ok=True) except PermissionError: fallback = Path(__file__).parent.parent / 'data' / 'tournaments.db' fallback.parent.mkdir(parents=True, exist_ok=True) p = fallback conn = sqlite3.connect(str(p)) conn.execute('PRAGMA journal_mode=WAL') conn.execute(''' CREATE TABLE IF NOT EXISTS subscriptions ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, fide_id INTEGER NOT NULL, player_name TEXT NOT NULL, rating INTEGER DEFAULT 0, fed TEXT DEFAULT '', lang TEXT DEFAULT 'en', tournament_url TEXT NOT NULL, tournament_name TEXT DEFAULT '', player_sno INTEGER DEFAULT 0, last_results_count INTEGER DEFAULT 0, last_points REAL DEFAULT 0.0, last_round_done INTEGER DEFAULT 0, start_date TEXT DEFAULT '', end_date TEXT DEFAULT '', active INTEGER DEFAULT 1, UNIQUE(user_id, fide_id, tournament_url) ) ''') conn.execute(''' CREATE TABLE IF NOT EXISTS tnr_state ( key TEXT PRIMARY KEY, value TEXT ) ''') conn.execute(''' CREATE TABLE IF NOT EXISTS tnr_cache ( tnr INTEGER PRIMARY KEY, name TEXT DEFAULT '', start_date TEXT DEFAULT '', end_date TEXT DEFAULT '', players_json TEXT DEFAULT '{}', scanned_at TEXT DEFAULT (datetime('now')) ) ''') # Migrate existing tables that may be missing columns added in later versions _migrate_add_column(conn, 'subscriptions', 'start_date', "TEXT DEFAULT ''") _migrate_add_column(conn, 'subscriptions', 'end_date', "TEXT DEFAULT ''") conn.commit() return conn # ═══ FIDE profile scraper ═══ def fetch_fide_player(fide_id: int) -> dict: url = FIDE_URL.format(fide_id=fide_id) resp = requests.get(url, headers=HEADERS, timeout=20) if resp.status_code != 200: raise RuntimeError(f'FIDE profile not found (HTTP {resp.status_code})') html = resp.text # Check for "No record found" if 'No record found' in html: raise RuntimeError(f'Player with FIDE ID {fide_id} not found') # Name:

Carlsen, Magnus

name = '' m = re.search(r']*>\s*(.*?)\s*', html) if m: name = m.group(1).strip() name = re.sub(r'<[^>]+>', '', name).strip() if not name: raise RuntimeError('Could not extract player name from FIDE profile') # Federation: "National Rank NOR" fed = '' m = re.search(r'National Rank\s+([A-Z]{3})', html) if m: fed = m.group(1) # Rating (standard):

2841

STANDARD rating = 0 m = re.search(r'

(\d{3,4})

\s*]*>\s*STANDARD', html) if m: rating = int(m.group(1)) # Title:

Grandmaster

title = '' m = re.search(r'\s*

(.*?)

', html) if m: title = m.group(1).strip() # Some players have multiple titles in adjacent

— take only the first one title = re.sub(r'<[^>]+>', '', title).strip() # Canonical FIDE ID from profile page:

1503014

m = re.search(r'\s*(\d+)\s*

', html) if m: fide_id = int(m.group(1)) return { 'fide_id': fide_id, 'name': name, 'rating': rating, 'fed': fed, 'title': title, } # ═══ Subscription CRUD ═══ def add_subscription(user_id: int, player: dict, tournament_url: str, tournament_name: str, player_sno: int, last_results_count: int, last_points: float, lang: str, start_date: str = '', end_date: str = ''): conn = _get_conn() conn.execute(''' INSERT OR REPLACE INTO subscriptions (user_id, fide_id, player_name, rating, fed, lang, tournament_url, tournament_name, player_sno, last_results_count, last_points, last_round_done, start_date, end_date, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, 1) ''', (user_id, player['fide_id'], player['name'], player['rating'], player.get('fed', ''), lang, tournament_url, tournament_name, player_sno, last_results_count, last_points, start_date, end_date)) conn.commit() conn.close() def get_user_subs(user_id: int) -> list: conn = _get_conn() rows = conn.execute( 'SELECT id, player_name, rating, fed, tournament_name, ' 'last_points, last_round_done, active FROM subscriptions ' 'WHERE user_id = ? AND active = 1 ORDER BY id', (user_id,)).fetchall() conn.close() return [{'id': r[0], 'player_name': r[1], 'rating': r[2], 'fed': r[3], 'tournament_name': r[4], 'last_points': r[5], 'last_round_done': r[6], 'active': r[7]} for r in rows] def get_active_subs() -> list: conn = _get_conn() rows = conn.execute( 'SELECT id, user_id, fide_id, player_name, rating, fed, lang, ' 'tournament_url, tournament_name, player_sno, last_results_count, ' 'last_points, last_round_done FROM subscriptions ' 'WHERE active = 1').fetchall() conn.close() return [dict(zip( ['id', 'user_id', 'fide_id', 'player_name', 'rating', 'fed', 'lang', 'tournament_url', 'tournament_name', 'player_sno', 'last_results_count', 'last_points', 'last_round_done'], r)) for r in rows] def remove_subscription(sub_id: int, user_id: int) -> bool: conn = _get_conn() cur = conn.execute( 'UPDATE subscriptions SET active = 0 WHERE id = ? AND user_id = ?', (sub_id, user_id)) conn.commit() deleted = cur.rowcount > 0 conn.close() return deleted def update_result(sub_id: int, results_count: int, points: float): conn = _get_conn() conn.execute( 'UPDATE subscriptions SET last_results_count = ?, last_points = ? ' 'WHERE id = ?', (results_count, points, sub_id)) conn.commit() conn.close() def update_round(sub_id: int, round_num: int): conn = _get_conn() conn.execute( 'UPDATE subscriptions SET last_round_done = ? WHERE id = ?', (round_num, sub_id)) conn.commit() conn.close() # ═══ Tournament player lookup ═══ def _normalize(s: str) -> str: return re.sub(r'\s+', ' ', s.strip()).lower() def find_player(tournament: dict, name: str, sno: int = 0) -> dict | None: standings = tournament.get('standings', []) # Try exact SNo match first if sno: for s in standings: if s.get('starting_sno') == sno: return s # Try exact name match (normalized) norm_target = _normalize(name) for s in standings: if _normalize(s['name']) == norm_target: return s # Fuzzy: split into parts, find best overlap target_parts = set(norm_target.split()) best, best_overlap = None, 0 for s in standings: s_parts = set(_normalize(s['name']).split()) overlap = len(target_parts & s_parts) if overlap > best_overlap: best_overlap = overlap best = s if best and best_overlap >= 1: return best return None def find_opponent_name(tournament: dict, opp_sno: int) -> str: if not opp_sno: return 'BYE' standings = tournament.get('standings', []) for s in standings: if s.get('starting_sno') == opp_sno: return s['name'] return f'#{opp_sno}' # ═══ Result formatting ═══ def _md_escape_tracker(text: str) -> str: escape_chars = r'_*[]()~`>#+-=|{}.!' result = [] for ch in text: if ch in escape_chars: result.append('\\' + ch) else: result.append(ch) return ''.join(result) def format_result(player_name: str, result: dict, opponent_name: str, points: float, lang: str) -> str: loc = RESULT_LOCALE.get(lang, RESULT_LOCALE['en']) score = result.get('score', 0.0) color = result.get('color', '') rd = result.get('round', '?') score_label = {1.0: loc['win'], 0.5: loc['draw'], 0.0: loc['loss']} score_desc = score_label.get(score, f'{score}') color_desc = loc['white'] if color == 'w' else loc['black'] safe_name = _md_escape_tracker(player_name) safe_opp = _md_escape_tracker(opponent_name) safe_points = _md_escape_tracker(f'{points:.1f}') return ( f'♟ *{safe_name}* — {loc["round"]} {rd}\n' f'{score_desc} \({color_desc}\) {loc["vs"]} {safe_opp}\n' f'{loc["points"]}: {points:.1f}' ) # ═══ TNR state cache ═══ DEFAULT_MAX_TNR = 500000 def get_tnr_state(key: str, default: str = '') -> str: conn = _get_conn() row = conn.execute( 'SELECT value FROM tnr_state WHERE key = ?', (key,)).fetchone() conn.close() return row[0] if row else default def set_tnr_state(key: str, value: str): conn = _get_conn() conn.execute( 'INSERT OR REPLACE INTO tnr_state (key, value) VALUES (?, ?)', (key, value)) conn.commit() conn.close() # ═══ TNR cache (avoids re-fetching art=0 pages) ═══ def _get_cached_tnr(tnr: int) -> dict | None: conn = _get_conn() row = conn.execute( 'SELECT name, start_date, end_date, players_json FROM tnr_cache WHERE tnr = ?', (tnr,)).fetchone() conn.close() if row is None: return None players_raw = json.loads(row[3]) players = {int(k): v for k, v in players_raw.items()} return { 'name': row[0], 'start_date': row[1], 'end_date': row[2], 'players': players, } def _save_tnr_cache(tnr: int, data: dict): players_raw = {str(k): v for k, v in data.get('players', {}).items()} conn = _get_conn() conn.execute( 'INSERT OR REPLACE INTO tnr_cache (tnr, name, start_date, end_date, players_json) ' 'VALUES (?, ?, ?, ?, ?)', (tnr, data.get('name', ''), data.get('start_date', ''), data.get('end_date', ''), json.dumps(players_raw, ensure_ascii=False))) conn.commit() conn.close() def _fetch_and_parse_art0(tnr: int) -> dict | None: """Fetch art=0 page (or use cache), return parsed dict or None.""" cached = _get_cached_tnr(tnr) if cached is not None: return cached html = _fetch_art0(tnr) if html is None: return None parsed = _parse_art0_page(html) _save_tnr_cache(tnr, parsed) return parsed # ═══ Tournament scanner (art=0 pages) ═══ ART0_URL = 'https://chess-results.com/tnr{tnr}.aspx?lan=11&art=0&turdet=YES' def _parse_art0_page(html: str) -> dict: """Parse art=0 page: extract tournament name, dates, and list of players.""" from bs4 import BeautifulSoup result = { 'name': '', 'start_date': '', 'end_date': '', 'players': {}, # fide_id -> {sno, name, rating, fed} } soup = BeautifulSoup(html, 'html.parser') # Tournament name: from

h2 = soup.find('h2') if h2: result['name'] = h2.get_text(strip=True) # Dates: look for table row with "Дата(ы)" label for tr in soup.find_all('tr'): tds = tr.find_all('td') texts = [td.get_text(strip=True) for td in tds] for i, t in enumerate(texts): if 'Дата' in t and i + 1 < len(texts): m = re.match( r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})', texts[i + 1]) if m: result['start_date'] = m.group(1).replace('/', '-') result['end_date'] = m.group(2).replace('/', '-') break if 'Date' in t and '(' in t and i + 1 < len(texts): m = re.match( r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})', texts[i + 1]) if m: result['start_date'] = m.group(1).replace('/', '-') result['end_date'] = m.group(2).replace('/', '-') break # Players: find the main table with SNo/Name/FIDE/FED/Rating columns for table in soup.find_all('table'): rows = table.find_all('tr') player_rows = 0 for row in rows: cells = row.find_all('td') texts = [c.get_text(strip=True) for c in cells] if len(texts) >= 4 and texts[0].isdigit() and re.search(r'[A-Z]{3}', ' '.join(texts)): player_rows += 1 if player_rows < 5: continue for row in rows: cells = row.find_all('td') texts = [c.get_text(strip=True) for c in cells] if len(texts) < 4 or not texts[0].isdigit(): continue try: sno = int(texts[0]) except ValueError: continue # Find FIDE ID column: numeric, 6-10 digits, near a FED column fide_id = 0 fed = '' rating = 0 name = '' for ci, t in enumerate(texts): if re.match(r'^\d{6,10}$', t): fide_id = int(t) if ci + 1 < len(texts) and re.match(r'^[A-Z]{3}$', texts[ci + 1]): fed = texts[ci + 1] if ci + (2 if fed else 1) < len(texts): rt = texts[ci + (2 if fed else 1)] if rt.isdigit(): rating = int(rt) if ci >= 1 and re.search(r'[A-Za-zА-Яа-я]', t) and len(t) > 2: if not name: name = t if fide_id: result['players'][fide_id] = { 'sno': sno, 'name': name, 'rating': rating, 'fed': fed, } if result['players']: break return result def _fetch_art0(tnr: int) -> str | None: """Fetch art=0 page, return HTML or None if not a valid tournament.""" import time url = ART0_URL.format(tnr=tnr) for attempt in range(3): try: resp = requests.get(url, headers=HEADERS, timeout=15) if resp.status_code == 429: time.sleep(2 * (attempt + 1)) continue if resp.status_code != 200: return None resp.encoding = 'utf-8' html = resp.text # chess-results returns 200 even for non-existent tnrs # Valid tournaments have

and are >30KB if ' int: """Find the current maximum tnr by probing upwards from saved value.""" import time saved = get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR)) tnr = int(saved) for step in [10000, 1000, 100, 50, 20, 10, 5, 1]: while True: parsed = _fetch_and_parse_art0(tnr + step) if parsed is not None: tnr += step else: break time.sleep(0.3) if tnr > int(saved): set_tnr_state('max_tnr_seen', str(tnr)) return tnr def scan_for_player(fide_id: int, max_tnr: int = None, limit: int = 200) -> list: """Scan tournaments for a player by FIDE ID. Scans multiple TNR windows since chess-results numbering has sparse clusters. Returns list of {tnr, url, name, start_date, end_date, sno, player_name, fide_id}. """ import time results = [] seen_tnrs = set() # Phase 1: check tnr_cache via SQL (single query, instant). Return if found. conn = _get_conn() pattern = f'%"{fide_id}"%' rows = conn.execute( 'SELECT tnr, name, start_date, end_date, players_json FROM tnr_cache ' 'WHERE tnr >= ? AND tnr <= ? AND players_json LIKE ? ' 'ORDER BY tnr DESC', (1434000, 1450000, pattern)).fetchall() conn.close() for row in rows: tnr, tname, tstart, tend, players_json = row players = {int(k): v for k, v in json.loads(players_json).items()} player_info = players.get(fide_id) if player_info is None: continue results.append({ 'tnr': tnr, 'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11', 'name': tname or f'Tournament {tnr}', 'start_date': tstart, 'end_date': tend, 'sno': player_info['sno'], 'player_name': player_info['name'], 'fide_id': fide_id, }) if results: set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results))) return results # Phase 2: cache miss — fall back to sequential TNR scan saved = int(get_tnr_state('max_tnr_seen', '0')) if max_tnr is None: max_tnr = max(saved, discover_max_tnr()) windows = [(max_tnr, limit)] for start, win_limit in windows: for tnr in range(start, max(1, start - win_limit), -1): if tnr in seen_tnrs: continue seen_tnrs.add(tnr) parsed = _fetch_and_parse_art0(tnr) if parsed is None: continue if not parsed['players']: continue player_info = parsed['players'].get(fide_id) if player_info is None: continue results.append({ 'tnr': tnr, 'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11', 'name': parsed['name'] or f'Tournament {tnr}', 'start_date': parsed['start_date'], 'end_date': parsed['end_date'], 'sno': player_info['sno'], 'player_name': player_info['name'], 'fide_id': fide_id, }) time.sleep(0.3) if results: set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results))) elif max_tnr > saved: set_tnr_state('max_tnr_seen', str(max_tnr)) return results def get_unique_fide_ids() -> list: """Return unique FIDE IDs from active subscriptions.""" conn = _get_conn() rows = conn.execute( 'SELECT DISTINCT fide_id FROM subscriptions WHERE active = 1').fetchall() conn.close() return [r[0] for r in rows] def check_new_tournaments_for_player(fide_id: int, new_max_tnr: int, prev_max_tnr: int) -> list: """Check new tnr range for a specific player. Returns new tournaments found.""" results = [] for tnr in range(new_max_tnr, prev_max_tnr, -1): parsed = _fetch_and_parse_art0(tnr) if parsed is None: continue if not parsed['players']: continue player_info = parsed['players'].get(fide_id) if player_info is None: continue url = f'https://chess-results.com/tnr{tnr}.aspx?lan=11' results.append({ 'tnr': tnr, 'url': url, 'name': parsed['name'] or f'Tournament {tnr}', 'start_date': parsed['start_date'], 'end_date': parsed['end_date'], 'sno': player_info['sno'], 'player_name': player_info['name'], 'fide_id': fide_id, }) return results async def warmup_cache(context): """Build full tnr_cache index (runs once at startup, then every 6h).""" import asyncio, sys, traceback # Skip if warmup already completed or cache already populated status = get_tnr_state('warmup_tnr', '') if status == 'done': return conn = _get_conn() cache_count = conn.execute('SELECT COUNT(*) FROM tnr_cache').fetchone()[0] conn.close() if cache_count > 1000: set_tnr_state('warmup_tnr', 'done') return print('warmup_cache: starting cluster scan (step=1, 1445000→1434000)...', file=sys.stderr) try: await asyncio.to_thread(_warmup_cache_sync) print('warmup_cache: cluster scan complete', file=sys.stderr) except Exception as e: print(f'warmup_cache: FAILED: {e}', file=sys.stderr) traceback.print_exc(file=sys.stderr) def _warmup_cache_sync(): """Scan the 1.4M cluster at step 1, caching ALL valid TNRs with players. One-time cost ~40 min, then all searches are instant SQL queries. Saves progress to tnr_state so it resumes after restart.""" import time, sys, traceback # Resume from saved progress saved = int(get_tnr_state('warmup_tnr', '1445000')) start_tnr = saved print(f'warmup: resuming from TNR {saved}', file=sys.stderr) for tnr in range(start_tnr, 1433999, -1): try: _fetch_and_parse_art0(tnr) except Exception as e: print(f'warmup: error at TNR {tnr}: {e}', file=sys.stderr) traceback.print_exc(file=sys.stderr) time.sleep(2) continue if tnr % 100 == 0: set_tnr_state('warmup_tnr', str(tnr)) if tnr % 500 == 0: elapsed = (start_tnr - tnr) // 500 * 500 # rough print(f'warmup: progress TNR {tnr} (started at {start_tnr})', file=sys.stderr) time.sleep(0.02) # Mark warmup as complete — next run will see this and skip set_tnr_state('warmup_tnr', 'done') async def check_all_subscriptions(context): from swiss_calc.parser import fetch_tournament from swiss_calc.swiss import calculate_next_round from bots.client_bot import format_pairings, _render_chunks from telegram.constants import ParseMode subs = get_active_subs() if not subs: return today_str = date.today().isoformat() for sub in subs: try: tournament = fetch_tournament(sub['tournament_url']) except Exception: continue player = find_player(tournament, sub['player_name'], sub['player_sno']) if player is None: continue results_count = len(player.get('results', [])) points = player.get('points', 0.0) # New individual result detected if results_count > sub['last_results_count']: new_r = player['results'][-1] opp_name = find_opponent_name(tournament, new_r.get('opponent', 0)) msg = format_result( player['name'], new_r, opp_name, points, sub['lang']) try: await context.bot.send_message( sub['user_id'], msg, parse_mode=ParseMode.MARKDOWN_V2) except Exception: pass update_result(sub['id'], results_count, points) # Round fully completed → calculate next round pairings current_rd = tournament.get('current_round', 0) if current_rd > sub['last_round_done'] and current_rd < tournament.get('num_rounds', 0): try: result = calculate_next_round(tournament) fmt = format_pairings(result, tournament.get('name', ''), sub['lang'], sub['player_name'], sub['player_sno']) chunks = _render_chunks(fmt) for chunk in chunks: try: await context.bot.send_message( sub['user_id'], chunk, parse_mode=ParseMode.MARKDOWN_V2, disable_web_page_preview=True) except Exception: break except Exception: pass update_round(sub['id'], current_rd) async def rescan_new_tournaments(context): """Hourly: discover new tnrs and check if subscribed players appear.""" prev_max = int(get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR))) new_max = discover_max_tnr() if new_max <= prev_max: return fide_ids = get_unique_fide_ids() if not fide_ids: return conn = _get_conn() user_fide_map = {} rows = conn.execute( 'SELECT DISTINCT user_id, fide_id, lang FROM subscriptions WHERE active = 1' ).fetchall() conn.close() for r in rows: user_fide_map.setdefault(r[1], []).append((r[0], r[2])) for fide_id in fide_ids: new_tournaments = check_new_tournaments_for_player( fide_id, new_max, prev_max) for t in new_tournaments: for user_id, lang in user_fide_map.get(fide_id, []): player = {'fide_id': fide_id, 'name': t['player_name'], 'rating': 0, 'fed': ''} add_subscription( user_id, player, t['url'], t['name'], t['sno'], 0, 0.0, lang, t['start_date'], t['end_date']) try: safe_name = _md_escape_tracker(t['player_name']) safe_tour = _md_escape_tracker(t['name']) dates = '' if t['start_date'] and t['end_date']: dates = f'\\({t["start_date"]} — {t["end_date"]}\\)' await context.bot.send_message( user_id, f'🆕 *{safe_name}* найден в новом турнире\!\n' f'📅 *{safe_tour}*{dates}\n' 'Автоматически подписал — буду отслеживать\.', parse_mode=ParseMode.MARKDOWN_V2) except Exception: pass set_tnr_state('max_tnr_seen', str(new_max))