ChessCalcNextTour/bots/tracker.py

"""
Player tracking: FIDE profile lookup, subscription management,
background polling for new results and pairings.
"""
import json
import re
import os
import sys
import sqlite3
import requests
from pathlib import Path
from datetime import date

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

DB_PATH = os.environ.get('STATS_DB', '/app/data/tournaments.db')
FIDE_URL = 'https://ratings.fide.com/profile/{fide_id}'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
}

RESULT_LOCALE = {
    'ru': {
        'win': 'победа', 'draw': 'ничья', 'loss': 'поражение',
        'white': 'белыми', 'black': 'чёрными',
        'points': 'Очков', 'round': 'тур', 'vs': 'против',
    },
    'en': {
        'win': 'win', 'draw': 'draw', 'loss': 'loss',
        'white': 'White', 'black': 'Black',
        'points': 'Points', 'round': 'round', 'vs': 'vs',
    },
}


def _migrate_add_column(conn, table: str, column: str, col_def: str):
    cols = [r[1] for r in conn.execute(f'PRAGMA table_info({table})').fetchall()]
    if column not in cols:
        conn.execute(f'ALTER TABLE {table} ADD COLUMN {column} {col_def}')


def _get_conn():
    p = Path(DB_PATH)
    try:
        p.parent.mkdir(parents=True, exist_ok=True)
    except PermissionError:
        fallback = Path(__file__).parent.parent / 'data' / 'tournaments.db'
        fallback.parent.mkdir(parents=True, exist_ok=True)
        p = fallback
    conn = sqlite3.connect(str(p))
    conn.execute('PRAGMA journal_mode=WAL')
    conn.execute('''
        CREATE TABLE IF NOT EXISTS subscriptions (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            user_id INTEGER NOT NULL,
            fide_id INTEGER NOT NULL,
            player_name TEXT NOT NULL,
            rating INTEGER DEFAULT 0,
            fed TEXT DEFAULT '',
            lang TEXT DEFAULT 'en',
            tournament_url TEXT NOT NULL,
            tournament_name TEXT DEFAULT '',
            player_sno INTEGER DEFAULT 0,
            last_results_count INTEGER DEFAULT 0,
            last_points REAL DEFAULT 0.0,
            last_round_done INTEGER DEFAULT 0,
            start_date TEXT DEFAULT '',
            end_date TEXT DEFAULT '',
            active INTEGER DEFAULT 1,
            UNIQUE(user_id, fide_id, tournament_url)
        )
    ''')
    conn.execute('''
        CREATE TABLE IF NOT EXISTS tnr_state (
            key TEXT PRIMARY KEY,
            value TEXT
        )
    ''')
    conn.execute('''
        CREATE TABLE IF NOT EXISTS tnr_cache (
            tnr INTEGER PRIMARY KEY,
            name TEXT DEFAULT '',
            start_date TEXT DEFAULT '',
            end_date TEXT DEFAULT '',
            players_json TEXT DEFAULT '{}',
            scanned_at TEXT DEFAULT (datetime('now'))
        )
    ''')
    # Migrate existing tables that may be missing columns added in later versions
    _migrate_add_column(conn, 'subscriptions', 'start_date', "TEXT DEFAULT ''")
    _migrate_add_column(conn, 'subscriptions', 'end_date', "TEXT DEFAULT ''")
    conn.commit()
    return conn


# ═══ FIDE profile scraper ═══

def fetch_fide_player(fide_id: int) -> dict:
    url = FIDE_URL.format(fide_id=fide_id)
    resp = requests.get(url, headers=HEADERS, timeout=20)
    if resp.status_code != 200:
        raise RuntimeError(f'FIDE profile not found (HTTP {resp.status_code})')
    html = resp.text

    # Check for "No record found"
    if 'No record found' in html:
        raise RuntimeError(f'Player with FIDE ID {fide_id} not found')

    # Name: <h1 class="player-title">Carlsen, Magnus</h1>
    name = ''
    m = re.search(r'<h1\s+class="player-title"[^>]*>\s*(.*?)\s*</h1>', html)
    if m:
        name = m.group(1).strip()
        name = re.sub(r'<[^>]+>', '', name).strip()
    if not name:
        raise RuntimeError('Could not extract player name from FIDE profile')

    # Federation: "National Rank NOR"
    fed = ''
    m = re.search(r'National Rank\s+([A-Z]{3})', html)
    if m:
        fed = m.group(1)

    # Rating (standard): <p>2841</p><p ...>STANDARD
    rating = 0
    m = re.search(r'<p>(\d{3,4})</p>\s*<p[^>]*>\s*STANDARD', html)
    if m:
        rating = int(m.group(1))

    # Title: <div class="profile-info-title "><p>Grandmaster</p>
    title = ''
    m = re.search(r'<div\s+class="profile-info-title[^"]*"\s*>\s*<p>(.*?)</p>', html)
    if m:
        title = m.group(1).strip()
        # Some players have multiple titles in adjacent <p> — take only the first one
        title = re.sub(r'<[^>]+>', '', title).strip()

    # Canonical FIDE ID from profile page: <p class="profile-info-id ">1503014</p>
    m = re.search(r'<p\s+class="profile-info-id\s*">\s*(\d+)\s*</p>', html)
    if m:
        fide_id = int(m.group(1))

    return {
        'fide_id': fide_id, 'name': name, 'rating': rating,
        'fed': fed, 'title': title,
    }


# ═══ Subscription CRUD ═══

def add_player_watch(user_id: int, player: dict, lang: str):
    """Subscribe to a player with no active tournament yet.

    rescan_new_tournaments will auto-upgrade this to a real subscription
    when a tournament appears.
    """
    conn = _get_conn()
    conn.execute('''
        INSERT OR IGNORE INTO subscriptions
        (user_id, fide_id, player_name, rating, fed, lang, tournament_url,
         tournament_name, player_sno, last_results_count, last_points,
         last_round_done, active)
        VALUES (?, ?, ?, ?, ?, ?, 'fide_only', '', 0, 0, 0.0, 0, 1)
    ''', (user_id, player['fide_id'], player['name'], player.get('rating', 0),
          player.get('fed', ''), lang))
    conn.commit()
    conn.close()


def add_subscription(user_id: int, player: dict, tournament_url: str,
                     tournament_name: str, player_sno: int,
                     last_results_count: int, last_points: float,
                     lang: str, start_date: str = '', end_date: str = ''):
    conn = _get_conn()
    conn.execute('''
        INSERT OR REPLACE INTO subscriptions
        (user_id, fide_id, player_name, rating, fed, lang, tournament_url,
         tournament_name, player_sno, last_results_count, last_points,
         last_round_done, start_date, end_date, active)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, 1)
    ''', (user_id, player['fide_id'], player['name'], player['rating'],
          player.get('fed', ''), lang, tournament_url, tournament_name,
          player_sno, last_results_count, last_points, start_date, end_date))
    conn.commit()
    conn.close()


def get_user_subs(user_id: int) -> list:
    """Return subscriptions grouped by player (fide_id)."""
    conn = _get_conn()
    rows = conn.execute(
        'SELECT fide_id, player_name, rating, fed, tournament_name, '
        'last_points, last_round_done, tournament_url FROM subscriptions '
        'WHERE user_id = ? AND active = 1 ORDER BY fide_id, id',
        (user_id,)).fetchall()
    conn.close()

    players: dict = {}
    for r in rows:
        fide_id, pname, rating, fed, tname, pts, rd, url = r
        if fide_id not in players:
            players[fide_id] = {
                'fide_id': fide_id, 'player_name': pname,
                'rating': rating, 'fed': fed or '',
                'tournaments': [],
            }
        if url == 'fide_only':
            players[fide_id]['tournaments'].append({
                'name': None, 'points': pts, 'round_done': rd,
            })
        else:
            players[fide_id]['tournaments'].append({
                'name': tname or url, 'points': pts, 'round_done': rd,
            })
    return list(players.values())


def get_active_subs() -> list:
    conn = _get_conn()
    rows = conn.execute(
        'SELECT id, user_id, fide_id, player_name, rating, fed, lang, '
        'tournament_url, tournament_name, player_sno, last_results_count, '
        'last_points, last_round_done FROM subscriptions '
        'WHERE active = 1').fetchall()
    conn.close()
    return [dict(zip(
        ['id', 'user_id', 'fide_id', 'player_name', 'rating', 'fed', 'lang',
         'tournament_url', 'tournament_name', 'player_sno',
         'last_results_count', 'last_points', 'last_round_done'], r))
        for r in rows]


def remove_subscription(sub_id: int, user_id: int) -> bool:
    conn = _get_conn()
    cur = conn.execute(
        'UPDATE subscriptions SET active = 0 WHERE id = ? AND user_id = ?',
        (sub_id, user_id))
    conn.commit()
    deleted = cur.rowcount > 0
    conn.close()
    return deleted


def remove_player_subscriptions(fide_id: int, user_id: int) -> bool:
    """Deactivate all subscriptions for a player (by FIDE ID)."""
    conn = _get_conn()
    cur = conn.execute(
        'UPDATE subscriptions SET active = 0 WHERE fide_id = ? AND user_id = ?',
        (fide_id, user_id))
    conn.commit()
    deleted = cur.rowcount > 0
    conn.close()
    return deleted


def update_result(sub_id: int, results_count: int, points: float):
    conn = _get_conn()
    conn.execute(
        'UPDATE subscriptions SET last_results_count = ?, last_points = ? '
        'WHERE id = ?',
        (results_count, points, sub_id))
    conn.commit()
    conn.close()


def update_round(sub_id: int, round_num: int):
    conn = _get_conn()
    conn.execute(
        'UPDATE subscriptions SET last_round_done = ? WHERE id = ?',
        (round_num, sub_id))
    conn.commit()
    conn.close()


# ═══ Tournament player lookup ═══

def _normalize(s: str) -> str:
    return re.sub(r'\s+', ' ', s.strip()).lower()


def find_player(tournament: dict, name: str, sno: int = 0) -> dict | None:
    standings = tournament.get('standings', [])

    # Try exact SNo match first
    if sno:
        for s in standings:
            if s.get('starting_sno') == sno:
                return s

    # Try exact name match (normalized)
    norm_target = _normalize(name)
    for s in standings:
        if _normalize(s['name']) == norm_target:
            return s

    # Fuzzy: split into parts, find best overlap
    target_parts = set(norm_target.split())
    best, best_overlap = None, 0
    for s in standings:
        s_parts = set(_normalize(s['name']).split())
        overlap = len(target_parts & s_parts)
        if overlap > best_overlap:
            best_overlap = overlap
            best = s

    if best and best_overlap >= 1:
        return best

    return None


def find_opponent_name(tournament: dict, opp_sno: int) -> str:
    if not opp_sno:
        return 'BYE'
    standings = tournament.get('standings', [])
    for s in standings:
        if s.get('starting_sno') == opp_sno:
            return s['name']
    return f'#{opp_sno}'


# ═══ Result formatting ═══

def _md_escape_tracker(text: str) -> str:
    escape_chars = r'_*[]()~`>#+-=|{}.!'
    result = []
    for ch in text:
        if ch in escape_chars:
            result.append('\\' + ch)
        else:
            result.append(ch)
    return ''.join(result)


def format_result(player_name: str, result: dict, opponent_name: str,
                  points: float, lang: str) -> str:
    loc = RESULT_LOCALE.get(lang, RESULT_LOCALE['en'])
    score = result.get('score', 0.0)
    color = result.get('color', '')
    rd = result.get('round', '?')

    score_label = {1.0: loc['win'], 0.5: loc['draw'], 0.0: loc['loss']}
    score_desc = score_label.get(score, f'{score}')
    color_desc = loc['white'] if color == 'w' else loc['black']

    safe_name = _md_escape_tracker(player_name)
    safe_opp = _md_escape_tracker(opponent_name)
    safe_points = _md_escape_tracker(f'{points:.1f}')

    return (
        f'♟ *{safe_name}* — {loc["round"]} {rd}\n'
        f'{score_desc} \({color_desc}\) {loc["vs"]} {safe_opp}\n'
        f'{loc["points"]}: {points:.1f}'
    )


# ═══ TNR state cache ═══

DEFAULT_MAX_TNR = 500000


def get_tnr_state(key: str, default: str = '') -> str:
    conn = _get_conn()
    row = conn.execute(
        'SELECT value FROM tnr_state WHERE key = ?', (key,)).fetchone()
    conn.close()
    return row[0] if row else default


def set_tnr_state(key: str, value: str):
    conn = _get_conn()
    conn.execute(
        'INSERT OR REPLACE INTO tnr_state (key, value) VALUES (?, ?)',
        (key, value))
    conn.commit()
    conn.close()


# ═══ TNR cache (avoids re-fetching art=0 pages) ═══

def _get_cached_tnr(tnr: int) -> dict | None:
    conn = _get_conn()
    row = conn.execute(
        'SELECT name, start_date, end_date, players_json FROM tnr_cache WHERE tnr = ?',
        (tnr,)).fetchone()
    conn.close()
    if row is None:
        return None
    players_raw = json.loads(row[3])
    players = {int(k): v for k, v in players_raw.items()}
    return {
        'name': row[0], 'start_date': row[1], 'end_date': row[2],
        'players': players,
    }


def _save_tnr_cache(tnr: int, data: dict):
    players_raw = {str(k): v for k, v in data.get('players', {}).items()}
    conn = _get_conn()
    conn.execute(
        'INSERT OR REPLACE INTO tnr_cache (tnr, name, start_date, end_date, players_json) '
        'VALUES (?, ?, ?, ?, ?)',
        (tnr, data.get('name', ''), data.get('start_date', ''),
         data.get('end_date', ''), json.dumps(players_raw, ensure_ascii=False)))
    conn.commit()
    conn.close()


def _fetch_and_parse_art0(tnr: int) -> dict | None:
    """Fetch art=0 page (or use cache), return parsed dict or None."""
    cached = _get_cached_tnr(tnr)
    if cached is not None:
        return cached
    html = _fetch_art0(tnr)
    if html is None:
        return None
    parsed = _parse_art0_page(html)
    _save_tnr_cache(tnr, parsed)
    return parsed


def _refetch_art0(tnr: int) -> dict | None:
    """Always fetch art=0 via HTTP, update cache. Returns parsed dict or None."""
    html = _fetch_art0(tnr)
    if html is None:
        return None
    parsed = _parse_art0_page(html)
    _save_tnr_cache(tnr, parsed)
    return parsed


# ═══ Tournament scanner (art=0 pages) ═══

ART0_URL = 'https://chess-results.com/tnr{tnr}.aspx?lan=11&art=0&turdet=YES'


def _parse_art0_page(html: str) -> dict:
    """Parse art=0 page: extract tournament name, dates, and list of players."""
    from bs4 import BeautifulSoup

    result = {
        'name': '',
        'start_date': '',
        'end_date': '',
        'players': {},  # fide_id -> {sno, name, rating, fed}
    }

    soup = BeautifulSoup(html, 'html.parser')

    # Tournament name: from <h2>
    h2 = soup.find('h2')
    if h2:
        result['name'] = h2.get_text(strip=True)

    # Dates: look for table row with "Дата(ы)" label
    for tr in soup.find_all('tr'):
        tds = tr.find_all('td')
        texts = [td.get_text(strip=True) for td in tds]
        for i, t in enumerate(texts):
            if 'Дата' in t and i + 1 < len(texts):
                m = re.match(
                    r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})',
                    texts[i + 1])
                if m:
                    result['start_date'] = m.group(1).replace('/', '-')
                    result['end_date'] = m.group(2).replace('/', '-')
                    break
            if 'Date' in t and '(' in t and i + 1 < len(texts):
                m = re.match(
                    r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})',
                    texts[i + 1])
                if m:
                    result['start_date'] = m.group(1).replace('/', '-')
                    result['end_date'] = m.group(2).replace('/', '-')
                    break

    # Players: find the main table with SNo/Name/FIDE/FED/Rating columns
    for table in soup.find_all('table'):
        rows = table.find_all('tr')
        player_rows = 0
        for row in rows:
            cells = row.find_all('td')
            texts = [c.get_text(strip=True) for c in cells]
            if len(texts) >= 4 and texts[0].isdigit() and re.search(r'[A-Z]{3}', ' '.join(texts)):
                player_rows += 1
        if player_rows < 5:
            continue

        for row in rows:
            cells = row.find_all('td')
            texts = [c.get_text(strip=True) for c in cells]
            if len(texts) < 4 or not texts[0].isdigit():
                continue

            try:
                sno = int(texts[0])
            except ValueError:
                continue

            # Find FIDE ID column: numeric, 6-10 digits, near a FED column
            fide_id = 0
            fed = ''
            rating = 0
            name = ''
            for ci, t in enumerate(texts):
                if re.match(r'^\d{6,10}$', t):
                    fide_id = int(t)
                    if ci + 1 < len(texts) and re.match(r'^[A-Z]{3}$', texts[ci + 1]):
                        fed = texts[ci + 1]
                    if ci + (2 if fed else 1) < len(texts):
                        rt = texts[ci + (2 if fed else 1)]
                        if rt.isdigit():
                            rating = int(rt)
                if ci >= 1 and re.search(r'[A-Za-zА-Яа-я]', t) and len(t) > 2:
                    if not name:
                        name = t

            if fide_id:
                result['players'][fide_id] = {
                    'sno': sno, 'name': name, 'rating': rating, 'fed': fed,
                }

        if result['players']:
            break

    return result


def _fetch_art0(tnr: int) -> str | None:
    """Fetch art=0 page, return HTML or None if not a valid tournament."""
    import time
    url = ART0_URL.format(tnr=tnr)
    for attempt in range(3):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=15)
            if resp.status_code == 429:
                time.sleep(2 * (attempt + 1))
                continue
            if resp.status_code != 200:
                return None
            resp.encoding = 'utf-8'
            html = resp.text
            # chess-results returns 200 even for non-existent tnrs
            # Valid tournaments have <h2> and are >30KB
            if '<h2' not in html or len(html) < 30000:
                return None
            return html
        except requests.RequestException:
            time.sleep(1)
    return None


def discover_max_tnr() -> int:
    """Find the current maximum tnr by probing upwards from saved value."""
    import time
    saved = get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR))
    tnr = int(saved)

    # Use max cached TNR as a better starting point (avoids stuck-at-500K problem)
    conn = _get_conn()
    row = conn.execute('SELECT MAX(tnr) FROM tnr_cache').fetchone()
    conn.close()
    if row and row[0]:
        tnr = max(tnr, row[0])

    for step in [10000, 1000, 100, 50, 20, 10, 5, 1]:
        while True:
            parsed = _fetch_and_parse_art0(tnr + step)
            if parsed is not None:
                tnr += step
            else:
                break
            time.sleep(0.3)

    if tnr > int(saved):
        set_tnr_state('max_tnr_seen', str(tnr))

    return tnr


def scan_for_player(fide_id: int, max_tnr: int = None,
                    limit: int = 200) -> list:
    """Scan tournaments for a player by FIDE ID.

    Scans multiple TNR windows since chess-results numbering has sparse clusters.
    Returns list of {tnr, url, name, start_date, end_date, sno, player_name, fide_id}.
    """
    import time

    results = []
    seen_tnrs = set()

    # Phase 1: check full tnr_cache via SQL (single query, instant). Return if found.
    conn = _get_conn()
    pattern = f'%"{fide_id}"%'
    rows = conn.execute(
        'SELECT tnr, name, start_date, end_date, players_json FROM tnr_cache '
        'WHERE players_json LIKE ? '
        'ORDER BY tnr DESC',
        (pattern,)).fetchall()
    conn.close()
    for row in rows:
        tnr, tname, tstart, tend, players_json = row
        players = {int(k): v for k, v in json.loads(players_json).items()}
        player_info = players.get(fide_id)
        if player_info is None:
            continue
        results.append({
            'tnr': tnr,
            'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11',
            'name': tname or f'Tournament {tnr}',
            'start_date': tstart,
            'end_date': tend,
            'sno': player_info['sno'],
            'player_name': player_info['name'],
            'fide_id': fide_id,
        })
    if results:
        set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results)))
        return results

    # Phase 2: cache miss — fall back to sequential TNR scan
    saved = int(get_tnr_state('max_tnr_seen', '0'))
    if max_tnr is None:
        max_tnr = max(saved, discover_max_tnr())

    windows = [(max_tnr, limit)]

    for start, win_limit in windows:
        for tnr in range(start, max(1, start - win_limit), -1):
            if tnr in seen_tnrs:
                continue
            seen_tnrs.add(tnr)
            parsed = _fetch_and_parse_art0(tnr)
            if parsed is None:
                continue
            if not parsed['players']:
                continue
            player_info = parsed['players'].get(fide_id)
            if player_info is None:
                continue
            results.append({
                'tnr': tnr,
                'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11',
                'name': parsed['name'] or f'Tournament {tnr}',
                'start_date': parsed['start_date'],
                'end_date': parsed['end_date'],
                'sno': player_info['sno'],
                'player_name': player_info['name'],
                'fide_id': fide_id,
            })
            time.sleep(0.3)

    if results:
        set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results)))
    elif max_tnr > saved:
        set_tnr_state('max_tnr_seen', str(max_tnr))

    return results


def get_unique_fide_ids() -> list:
    """Return unique FIDE IDs from active subscriptions."""
    conn = _get_conn()
    rows = conn.execute(
        'SELECT DISTINCT fide_id FROM subscriptions WHERE active = 1').fetchall()
    conn.close()
    return [r[0] for r in rows]


def check_new_tournaments_for_player(fide_id: int, new_max_tnr: int,
                                     prev_max_tnr: int) -> list:
    """Check new tnr range for a specific player. Returns new tournaments found."""
    results = []
    for tnr in range(new_max_tnr, prev_max_tnr, -1):
        parsed = _fetch_and_parse_art0(tnr)
        if parsed is None:
            continue
        if not parsed['players']:
            continue
        player_info = parsed['players'].get(fide_id)
        if player_info is None:
            continue
        url = f'https://chess-results.com/tnr{tnr}.aspx?lan=11'
        results.append({
            'tnr': tnr, 'url': url,
            'name': parsed['name'] or f'Tournament {tnr}',
            'start_date': parsed['start_date'],
            'end_date': parsed['end_date'],
            'sno': player_info['sno'],
            'player_name': player_info['name'],
            'fide_id': fide_id,
        })
    return results


async def warmup_cache(context):
    """Build full tnr_cache index (runs once at startup, then every 6h)."""
    import asyncio, sys, traceback

    # Skip if warmup already completed or cache already populated
    status = get_tnr_state('warmup_tnr', '')
    if status == 'done':
        return
    conn = _get_conn()
    cache_count = conn.execute('SELECT COUNT(*) FROM tnr_cache').fetchone()[0]
    conn.close()
    if cache_count > 1000:
        set_tnr_state('warmup_tnr', 'done')
        return

    print('warmup_cache: starting cluster scan (step=1, 1445000→1434000)...', file=sys.stderr)
    try:
        await asyncio.to_thread(_warmup_cache_sync)
        print('warmup_cache: cluster scan complete', file=sys.stderr)
    except Exception as e:
        print(f'warmup_cache: FAILED: {e}', file=sys.stderr)
        traceback.print_exc(file=sys.stderr)


def _warmup_cache_sync():
    """Scan the 1.4M cluster at step 1, caching ALL valid TNRs with players.
    One-time cost ~40 min, then all searches are instant SQL queries.
    Saves progress to tnr_state so it resumes after restart."""
    import time, sys, traceback

    # Resume from saved progress
    saved = int(get_tnr_state('warmup_tnr', '1445000'))
    start_tnr = saved
    print(f'warmup: resuming from TNR {saved}', file=sys.stderr)

    for tnr in range(start_tnr, 1433999, -1):
        try:
            _fetch_and_parse_art0(tnr)
        except Exception as e:
            print(f'warmup: error at TNR {tnr}: {e}', file=sys.stderr)
            traceback.print_exc(file=sys.stderr)
            time.sleep(2)
            continue

        if tnr % 100 == 0:
            set_tnr_state('warmup_tnr', str(tnr))
        if tnr % 500 == 0:
            elapsed = (start_tnr - tnr) // 500 * 500  # rough
            print(f'warmup: progress TNR {tnr} (started at {start_tnr})', file=sys.stderr)
            time.sleep(0.02)

    # Mark warmup as complete — next run will see this and skip
    set_tnr_state('warmup_tnr', 'done')


async def rescan_existing_tournaments(context):
    """Every 2h: refetch art=0 for all cached TNRs, detect new players."""
    import time
    from bots.client_bot import _md_escape as _md_esc

    conn = _get_conn()
    cached_tnrs = [r[0] for r in conn.execute('SELECT tnr FROM tnr_cache ORDER BY tnr DESC').fetchall()]
    conn.close()
    if not cached_tnrs:
        return

    # Collect subscribed FIDE IDs and their users
    conn = _get_conn()
    sub_rows = conn.execute(
        'SELECT DISTINCT user_id, fide_id, lang FROM subscriptions WHERE active = 1'
    ).fetchall()
    conn.close()
    if not sub_rows:
        return
    fide_to_users = {}
    for user_id, fide_id, lang in sub_rows:
        fide_to_users.setdefault(fide_id, []).append((user_id, lang))

    for tnr in cached_tnrs:
        old_data = _get_cached_tnr(tnr)
        old_fids = set(old_data['players'].keys()) if old_data else set()

        new_data = _refetch_art0(tnr)
        if new_data is None:
            continue  # tournament deleted or unreachable
        new_fids = set(new_data['players'].keys())

        # New players that appeared since last cached snapshot
        appeared = new_fids - old_fids
        for fid in appeared:
            if fid not in fide_to_users:
                continue
            player_info = new_data['players'][fid]
            url = f'https://chess-results.com/tnr{tnr}.aspx?lan=11'
            for user_id, lang in fide_to_users[fid]:
                player = {'fide_id': fid, 'name': player_info['name'],
                          'rating': player_info.get('rating', 0), 'fed': ''}
                add_subscription(
                    user_id, player, url, new_data['name'] or f'Tournament {tnr}',
                    player_info['sno'], 0, 0.0, lang,
                    new_data.get('start_date', ''), new_data.get('end_date', ''))
                try:
                    safe_name = _md_esc(player_info['name'])
                    safe_tour = _md_esc(new_data['name'] or f'Tournament {tnr}')
                    dates = ''
                    if new_data.get('start_date') and new_data.get('end_date'):
                        dates = f'\\({new_data["start_date"]} — {new_data["end_date"]}\\)'
                    await context.bot.send_message(
                        user_id,
                        f'🆕 *{safe_name}* появился в турнире\!\n'
                        f'📅 *{safe_tour}*{dates}\n'
                        'Автоматически подписал — буду отслеживать\.',
                        parse_mode='MarkdownV2')
                except Exception:
                    pass
        time.sleep(0.1)


async def check_all_subscriptions(context):
    from swiss_calc.parser import fetch_tournament
    from swiss_calc.swiss import calculate_next_round
    from bots.client_bot import format_pairings, _render_chunks
    from telegram.constants import ParseMode

    subs = get_active_subs()
    if not subs:
        return

    today_str = date.today().isoformat()
    for sub in subs:
        if sub['tournament_url'] == 'fide_only':
            continue
        try:
            tournament = fetch_tournament(sub['tournament_url'])
        except Exception:
            continue

        player = find_player(tournament, sub['player_name'], sub['player_sno'])
        if player is None:
            continue

        results_count = len(player.get('results', []))
        points = player.get('points', 0.0)

        # New individual result detected
        if results_count > sub['last_results_count']:
            new_r = player['results'][-1]
            opp_name = find_opponent_name(tournament, new_r.get('opponent', 0))
            msg = format_result(
                player['name'], new_r, opp_name, points, sub['lang'])
            try:
                await context.bot.send_message(
                    sub['user_id'], msg, parse_mode=ParseMode.MARKDOWN_V2)
            except Exception:
                pass
            update_result(sub['id'], results_count, points)

        # Round fully completed → calculate next round pairings
        current_rd = tournament.get('current_round', 0)
        if current_rd > sub['last_round_done'] and current_rd < tournament.get('num_rounds', 0):
            try:
                result = calculate_next_round(tournament)
                fmt = format_pairings(result, tournament.get('name', ''), sub['lang'],
                                      sub['player_name'], sub['player_sno'])
                chunks = _render_chunks(fmt)
                for chunk in chunks:
                    try:
                        await context.bot.send_message(
                            sub['user_id'], chunk,
                            parse_mode=ParseMode.MARKDOWN_V2,
                            disable_web_page_preview=True)
                    except Exception:
                        break
            except Exception:
                pass
            update_round(sub['id'], current_rd)


async def rescan_new_tournaments(context):
    """Hourly: discover new tnrs and check if subscribed players appear."""
    prev_max = int(get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR)))
    new_max = discover_max_tnr()

    if new_max <= prev_max:
        return

    fide_ids = get_unique_fide_ids()
    if not fide_ids:
        return

    conn = _get_conn()
    user_fide_map = {}
    rows = conn.execute(
        'SELECT DISTINCT user_id, fide_id, lang FROM subscriptions WHERE active = 1'
    ).fetchall()
    conn.close()
    for r in rows:
        user_fide_map.setdefault(r[1], []).append((r[0], r[2]))

    for fide_id in fide_ids:
        new_tournaments = check_new_tournaments_for_player(
            fide_id, new_max, prev_max)
        for t in new_tournaments:
            for user_id, lang in user_fide_map.get(fide_id, []):
                player = {'fide_id': fide_id, 'name': t['player_name'],
                          'rating': 0, 'fed': ''}
                add_subscription(
                    user_id, player, t['url'], t['name'], t['sno'],
                    0, 0.0, lang, t['start_date'], t['end_date'])
                # Deactivate the fide_only watch entry now that we have a real tournament
                conn = _get_conn()
                conn.execute(
                    "UPDATE subscriptions SET active = 0 "
                    "WHERE user_id = ? AND fide_id = ? AND tournament_url = 'fide_only'",
                    (user_id, fide_id))
                conn.commit()
                conn.close()
                try:
                    safe_name = _md_escape_tracker(t['player_name'])
                    safe_tour = _md_escape_tracker(t['name'])
                    dates = ''
                    if t['start_date'] and t['end_date']:
                        dates = f'\\({t["start_date"]} — {t["end_date"]}\\)'
                    await context.bot.send_message(
                        user_id,
                        f'🆕 *{safe_name}* найден в новом турнире\!\n'
                        f'📅 *{safe_tour}*{dates}\n'
                        'Автоматически подписал — буду отслеживать\.',
                        parse_mode=ParseMode.MARKDOWN_V2)
                except Exception:
                    pass

    set_tnr_state('max_tnr_seen', str(new_max))