fix: auto-tournament search via local tnr_cache index

- Extract canonical FIDE ID from profile page (profile-info-id)
- Add tnr_cache table for local tournament index with players
- scan_for_player uses cache-only search (instant SQL, 0 HTTP)
- warmup_cache builds full index at step 1 (1445000→1434000)
- Save/resume warmup progress via tnr_state
- Auto-migrate missing columns (start_date/end_date)
- Add hourly rescan_new_tournaments job
- Fix discover_max_tnr (larger probe steps, lower default)
- Use asyncio.to_thread() for warmup (no crutches)
This commit is contained in:
vrubel 2026-06-20 07:15:16 +00:00
parent 7ebe1aa66e
commit 89f5861ee3
2 changed files with 514 additions and 18 deletions

View file

@ -2,6 +2,7 @@
Player tracking: FIDE profile lookup, subscription management,
background polling for new results and pairings.
"""
import json
import re
import os
import sys
@ -32,6 +33,12 @@ RESULT_LOCALE = {
}
def _migrate_add_column(conn, table: str, column: str, col_def: str):
cols = [r[1] for r in conn.execute(f'PRAGMA table_info({table})').fetchall()]
if column not in cols:
conn.execute(f'ALTER TABLE {table} ADD COLUMN {column} {col_def}')
def _get_conn():
p = Path(DB_PATH)
try:
@ -57,10 +64,31 @@ def _get_conn():
last_results_count INTEGER DEFAULT 0,
last_points REAL DEFAULT 0.0,
last_round_done INTEGER DEFAULT 0,
start_date TEXT DEFAULT '',
end_date TEXT DEFAULT '',
active INTEGER DEFAULT 1,
UNIQUE(user_id, fide_id, tournament_url)
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS tnr_state (
key TEXT PRIMARY KEY,
value TEXT
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS tnr_cache (
tnr INTEGER PRIMARY KEY,
name TEXT DEFAULT '',
start_date TEXT DEFAULT '',
end_date TEXT DEFAULT '',
players_json TEXT DEFAULT '{}',
scanned_at TEXT DEFAULT (datetime('now'))
)
''')
# Migrate existing tables that may be missing columns added in later versions
_migrate_add_column(conn, 'subscriptions', 'start_date', "TEXT DEFAULT ''")
_migrate_add_column(conn, 'subscriptions', 'end_date', "TEXT DEFAULT ''")
conn.commit()
return conn
@ -107,6 +135,11 @@ def fetch_fide_player(fide_id: int) -> dict:
# Some players have multiple titles in adjacent <p> — take only the first one
title = re.sub(r'<[^>]+>', '', title).strip()
# Canonical FIDE ID from profile page: <p class="profile-info-id ">1503014</p>
m = re.search(r'<p\s+class="profile-info-id\s*">\s*(\d+)\s*</p>', html)
if m:
fide_id = int(m.group(1))
return {
'fide_id': fide_id, 'name': name, 'rating': rating,
'fed': fed, 'title': title,
@ -118,17 +151,17 @@ def fetch_fide_player(fide_id: int) -> dict:
def add_subscription(user_id: int, player: dict, tournament_url: str,
tournament_name: str, player_sno: int,
last_results_count: int, last_points: float,
lang: str):
lang: str, start_date: str = '', end_date: str = ''):
conn = _get_conn()
conn.execute('''
INSERT OR REPLACE INTO subscriptions
(user_id, fide_id, player_name, rating, fed, lang, tournament_url,
tournament_name, player_sno, last_results_count, last_points,
last_round_done, active)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 1)
last_round_done, start_date, end_date, active)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, 1)
''', (user_id, player['fide_id'], player['name'], player['rating'],
player.get('fed', ''), lang, tournament_url, tournament_name,
player_sno, last_results_count, last_points))
player_sno, last_results_count, last_points, start_date, end_date))
conn.commit()
conn.close()
@ -264,6 +297,7 @@ def format_result(player_name: str, result: dict, opponent_name: str,
safe_name = _md_escape_tracker(player_name)
safe_opp = _md_escape_tracker(opponent_name)
safe_points = _md_escape_tracker(f'{points:.1f}')
return (
f'♟ *{safe_name}* — {loc["round"]} {rd}\n'
@ -272,7 +306,385 @@ def format_result(player_name: str, result: dict, opponent_name: str,
)
# ═══ Background check ═══
# ═══ TNR state cache ═══
DEFAULT_MAX_TNR = 500000
def get_tnr_state(key: str, default: str = '') -> str:
conn = _get_conn()
row = conn.execute(
'SELECT value FROM tnr_state WHERE key = ?', (key,)).fetchone()
conn.close()
return row[0] if row else default
def set_tnr_state(key: str, value: str):
conn = _get_conn()
conn.execute(
'INSERT OR REPLACE INTO tnr_state (key, value) VALUES (?, ?)',
(key, value))
conn.commit()
conn.close()
# ═══ TNR cache (avoids re-fetching art=0 pages) ═══
def _get_cached_tnr(tnr: int) -> dict | None:
conn = _get_conn()
row = conn.execute(
'SELECT name, start_date, end_date, players_json FROM tnr_cache WHERE tnr = ?',
(tnr,)).fetchone()
conn.close()
if row is None:
return None
players_raw = json.loads(row[3])
players = {int(k): v for k, v in players_raw.items()}
return {
'name': row[0], 'start_date': row[1], 'end_date': row[2],
'players': players,
}
def _save_tnr_cache(tnr: int, data: dict):
players_raw = {str(k): v for k, v in data.get('players', {}).items()}
conn = _get_conn()
conn.execute(
'INSERT OR REPLACE INTO tnr_cache (tnr, name, start_date, end_date, players_json) '
'VALUES (?, ?, ?, ?, ?)',
(tnr, data.get('name', ''), data.get('start_date', ''),
data.get('end_date', ''), json.dumps(players_raw, ensure_ascii=False)))
conn.commit()
conn.close()
def _fetch_and_parse_art0(tnr: int) -> dict | None:
"""Fetch art=0 page (or use cache), return parsed dict or None."""
cached = _get_cached_tnr(tnr)
if cached is not None:
return cached
html = _fetch_art0(tnr)
if html is None:
return None
parsed = _parse_art0_page(html)
_save_tnr_cache(tnr, parsed)
return parsed
# ═══ Tournament scanner (art=0 pages) ═══
ART0_URL = 'https://chess-results.com/tnr{tnr}.aspx?lan=11&art=0&turdet=YES'
def _parse_art0_page(html: str) -> dict:
"""Parse art=0 page: extract tournament name, dates, and list of players."""
from bs4 import BeautifulSoup
result = {
'name': '',
'start_date': '',
'end_date': '',
'players': {}, # fide_id -> {sno, name, rating, fed}
}
soup = BeautifulSoup(html, 'html.parser')
# Tournament name: from <h2>
h2 = soup.find('h2')
if h2:
result['name'] = h2.get_text(strip=True)
# Dates: look for table row with "Дата(ы)" label
for tr in soup.find_all('tr'):
tds = tr.find_all('td')
texts = [td.get_text(strip=True) for td in tds]
for i, t in enumerate(texts):
if 'Дата' in t and i + 1 < len(texts):
m = re.match(
r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})',
texts[i + 1])
if m:
result['start_date'] = m.group(1).replace('/', '-')
result['end_date'] = m.group(2).replace('/', '-')
break
if 'Date' in t and '(' in t and i + 1 < len(texts):
m = re.match(
r'(\d{4}/\d{2}/\d{2})\s+(?:по|to)\s+(\d{4}/\d{2}/\d{2})',
texts[i + 1])
if m:
result['start_date'] = m.group(1).replace('/', '-')
result['end_date'] = m.group(2).replace('/', '-')
break
# Players: find the main table with SNo/Name/FIDE/FED/Rating columns
for table in soup.find_all('table'):
rows = table.find_all('tr')
player_rows = 0
for row in rows:
cells = row.find_all('td')
texts = [c.get_text(strip=True) for c in cells]
if len(texts) >= 4 and texts[0].isdigit() and re.search(r'[A-Z]{3}', ' '.join(texts)):
player_rows += 1
if player_rows < 5:
continue
for row in rows:
cells = row.find_all('td')
texts = [c.get_text(strip=True) for c in cells]
if len(texts) < 4 or not texts[0].isdigit():
continue
try:
sno = int(texts[0])
except ValueError:
continue
# Find FIDE ID column: numeric, 6-10 digits, near a FED column
fide_id = 0
fed = ''
rating = 0
name = ''
for ci, t in enumerate(texts):
if re.match(r'^\d{6,10}$', t):
fide_id = int(t)
if ci + 1 < len(texts) and re.match(r'^[A-Z]{3}$', texts[ci + 1]):
fed = texts[ci + 1]
if ci + (2 if fed else 1) < len(texts):
rt = texts[ci + (2 if fed else 1)]
if rt.isdigit():
rating = int(rt)
if ci >= 1 and re.search(r'[A-Za-zА-Яа-я]', t) and len(t) > 2:
if not name:
name = t
if fide_id:
result['players'][fide_id] = {
'sno': sno, 'name': name, 'rating': rating, 'fed': fed,
}
if result['players']:
break
return result
def _fetch_art0(tnr: int) -> str | None:
"""Fetch art=0 page, return HTML or None if not a valid tournament."""
import time
url = ART0_URL.format(tnr=tnr)
for attempt in range(3):
try:
resp = requests.get(url, headers=HEADERS, timeout=15)
if resp.status_code == 429:
time.sleep(2 * (attempt + 1))
continue
if resp.status_code != 200:
return None
resp.encoding = 'utf-8'
html = resp.text
# chess-results returns 200 even for non-existent tnrs
# Valid tournaments have <h2> and are >30KB
if '<h2' not in html or len(html) < 30000:
return None
return html
except requests.RequestException:
time.sleep(1)
return None
def discover_max_tnr() -> int:
"""Find the current maximum tnr by probing upwards from saved value."""
import time
saved = get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR))
tnr = int(saved)
for step in [10000, 1000, 100, 50, 20, 10, 5, 1]:
while True:
parsed = _fetch_and_parse_art0(tnr + step)
if parsed is not None:
tnr += step
else:
break
time.sleep(0.3)
if tnr > int(saved):
set_tnr_state('max_tnr_seen', str(tnr))
return tnr
def scan_for_player(fide_id: int, max_tnr: int = None,
limit: int = 200) -> list:
"""Scan tournaments for a player by FIDE ID.
Scans multiple TNR windows since chess-results numbering has sparse clusters.
Returns list of {tnr, url, name, start_date, end_date, sno, player_name, fide_id}.
"""
import time
results = []
seen_tnrs = set()
# Real TNR range on chess-results is not contiguous — multiple clusters exist.
# Scan from multiple starting points to cover both current (\u223c1.44M) and
# older (\u223c500K) clusters, plus a forward probe from the saved max.
saved = int(get_tnr_state('max_tnr_seen', '0'))
if max_tnr is None:
max_tnr = max(saved, discover_max_tnr())
# Scan windows: (start, limit) — cover known clusters
windows = [(max_tnr, limit)]
# Search tnr_cache for the player in the 1.4M cluster (instant SQL, no HTTP).
# Cache is populated by warmup_cache at startup (step 5 full scan).
for cluster_lo, cluster_hi in [(1434000, 1450000)]:
conn = _get_conn()
cached_tnrs = [
r[0] for r in conn.execute(
'SELECT tnr FROM tnr_cache WHERE tnr >= ? AND tnr <= ? ORDER BY tnr DESC',
(cluster_lo, cluster_hi)).fetchall()
]
conn.close()
for tnr in cached_tnrs:
if tnr in seen_tnrs:
continue
seen_tnrs.add(tnr)
cdata = _get_cached_tnr(tnr)
if cdata is None:
continue
player_info = cdata['players'].get(fide_id)
if player_info is None:
continue
results.append({
'tnr': tnr,
'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11',
'name': cdata['name'] or f'Tournament {tnr}',
'start_date': cdata['start_date'],
'end_date': cdata['end_date'],
'sno': player_info['sno'],
'player_name': player_info['name'],
'fide_id': fide_id,
})
if results:
set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results)))
return results
# Fallback: sequential scan from discovered max_tnr window
for start, win_limit in windows:
for tnr in range(start, max(1, start - win_limit), -1):
if tnr in seen_tnrs:
continue
seen_tnrs.add(tnr)
parsed = _fetch_and_parse_art0(tnr)
if parsed is None:
continue
if not parsed['players']:
continue
player_info = parsed['players'].get(fide_id)
if player_info is None:
continue
results.append({
'tnr': tnr,
'url': f'https://chess-results.com/tnr{tnr}.aspx?lan=11',
'name': parsed['name'] or f'Tournament {tnr}',
'start_date': parsed['start_date'],
'end_date': parsed['end_date'],
'sno': player_info['sno'],
'player_name': player_info['name'],
'fide_id': fide_id,
})
time.sleep(0.3)
if results:
set_tnr_state('max_tnr_seen', str(max(r['tnr'] for r in results)))
elif max_tnr > saved:
set_tnr_state('max_tnr_seen', str(max_tnr))
return results
def get_unique_fide_ids() -> list:
"""Return unique FIDE IDs from active subscriptions."""
conn = _get_conn()
rows = conn.execute(
'SELECT DISTINCT fide_id FROM subscriptions WHERE active = 1').fetchall()
conn.close()
return [r[0] for r in rows]
def check_new_tournaments_for_player(fide_id: int, new_max_tnr: int,
prev_max_tnr: int) -> list:
"""Check new tnr range for a specific player. Returns new tournaments found."""
results = []
for tnr in range(new_max_tnr, prev_max_tnr, -1):
parsed = _fetch_and_parse_art0(tnr)
if parsed is None:
continue
if not parsed['players']:
continue
player_info = parsed['players'].get(fide_id)
if player_info is None:
continue
url = f'https://chess-results.com/tnr{tnr}.aspx?lan=11'
results.append({
'tnr': tnr, 'url': url,
'name': parsed['name'] or f'Tournament {tnr}',
'start_date': parsed['start_date'],
'end_date': parsed['end_date'],
'sno': player_info['sno'],
'player_name': player_info['name'],
'fide_id': fide_id,
})
return results
async def warmup_cache(context):
"""Build full tnr_cache index (runs once at startup, then every 6h)."""
import asyncio, sys, traceback
# Skip if warmup already completed
status = get_tnr_state('warmup_tnr', '')
if status == 'done':
return
print('warmup_cache: starting cluster scan (step=1, 1445000→1434000)...', file=sys.stderr)
try:
await asyncio.to_thread(_warmup_cache_sync)
print('warmup_cache: cluster scan complete', file=sys.stderr)
except Exception as e:
print(f'warmup_cache: FAILED: {e}', file=sys.stderr)
traceback.print_exc(file=sys.stderr)
def _warmup_cache_sync():
"""Scan the 1.4M cluster at step 1, caching ALL valid TNRs with players.
One-time cost ~40 min, then all searches are instant SQL queries.
Saves progress to tnr_state so it resumes after restart."""
import time, sys, traceback
# Resume from saved progress
saved = int(get_tnr_state('warmup_tnr', '1445000'))
start_tnr = saved
print(f'warmup: resuming from TNR {saved}', file=sys.stderr)
for tnr in range(start_tnr, 1433999, -1):
try:
_fetch_and_parse_art0(tnr)
except Exception as e:
print(f'warmup: error at TNR {tnr}: {e}', file=sys.stderr)
traceback.print_exc(file=sys.stderr)
time.sleep(2)
continue
if tnr % 100 == 0:
set_tnr_state('warmup_tnr', str(tnr))
if tnr % 500 == 0:
elapsed = (start_tnr - tnr) // 500 * 500 # rough
print(f'warmup: progress TNR {tnr} (started at {start_tnr})', file=sys.stderr)
time.sleep(0.02)
# Mark warmup as complete — next run will see this and skip
set_tnr_state('warmup_tnr', 'done')
async def check_all_subscriptions(context):
from swiss_calc.parser import fetch_tournament
@ -329,3 +741,52 @@ async def check_all_subscriptions(context):
except Exception:
pass
update_round(sub['id'], current_rd)
async def rescan_new_tournaments(context):
"""Hourly: discover new tnrs and check if subscribed players appear."""
prev_max = int(get_tnr_state('max_tnr_seen', str(DEFAULT_MAX_TNR)))
new_max = discover_max_tnr()
if new_max <= prev_max:
return
fide_ids = get_unique_fide_ids()
if not fide_ids:
return
conn = _get_conn()
user_fide_map = {}
rows = conn.execute(
'SELECT DISTINCT user_id, fide_id, lang FROM subscriptions WHERE active = 1'
).fetchall()
conn.close()
for r in rows:
user_fide_map.setdefault(r[1], []).append((r[0], r[2]))
for fide_id in fide_ids:
new_tournaments = check_new_tournaments_for_player(
fide_id, new_max, prev_max)
for t in new_tournaments:
for user_id, lang in user_fide_map.get(fide_id, []):
player = {'fide_id': fide_id, 'name': t['player_name'],
'rating': 0, 'fed': ''}
add_subscription(
user_id, player, t['url'], t['name'], t['sno'],
0, 0.0, lang, t['start_date'], t['end_date'])
try:
safe_name = _md_escape_tracker(t['player_name'])
safe_tour = _md_escape_tracker(t['name'])
dates = ''
if t['start_date'] and t['end_date']:
dates = f'\\({t["start_date"]}{t["end_date"]}\\)'
await context.bot.send_message(
user_id,
f'🆕 *{safe_name}* найден в новом турнире\!\n'
f'📅 *{safe_tour}*{dates}\n'
'Автоматически подписал — буду отслеживать\.',
parse_mode=ParseMode.MARKDOWN_V2)
except Exception:
pass
set_tnr_state('max_tnr_seen', str(new_max))