fix periodic-check request queue throughput bottleneck

The bot's request_queue.py 4s FIFO gate wasn't protecting against Lichess's
rate limiter — that's already handled downstream in LichessWebServices/
rate_limiter.py (0.2s, shared across all callers of our stats service). The
bot-side gate only paced calls to our own local service, and since it awaited
each request to full completion before dequeuing the next, real dispatch gaps
were max(4s, previous request's duration) — with 454 tracked gamer/user pairs,
any burst (e.g. after a restart) piled into the queue and took 10-20+ minutes
to drain.

Replace it with a paced-dispatch + bounded-concurrency design: a hard 2s floor
between dispatches (still never lets 2+ requests through in that window),
decoupled from completion time, with up to 10 requests actually in flight at
once via a semaphore. Doesn't touch the real Lichess-facing rate limit at all.

Also add deterministic per-(user,gamer) checkpoint jitter: previously every
pair sharing the same period_minutes re-locked onto the same wall-clock phase
on every restart (backlog collapse snaps period_end_approx to `now` for
everyone overdue at once), recreating the pileup each time. Jitter is stable
across restarts (crc32-based, not Python's salted hash()) and capped well
under the 2h stale-backlog threshold. Small startup stagger added too, purely
cosmetic smoothing on top of the jitter fix.

LichessClientTG_bot/request_queue.py

@@ -1,10 +1,17 @@
 """
 Request Queue for managing API requests with rate limiting
-Ensures minimum delay between requests to avoid DDoS and rate limiting
+
+Paces how fast we dispatch requests to our own local stats API
+(http://localhost:8002), and caps how many of those requests may be in
+flight at once. This does NOT protect against Lichess's real rate limiter —
+that protection already lives downstream, in LichessWebServices/rate_limiter.py
+(0.2s min delay, single shared instance, applied before every actual
+lichess.org call, shared by ALL callers of stats_service — not just this
+bot). These numbers only govern bot -> local-service traffic.
 """
 import asyncio
 import logging
-from typing import Callable, Any, Optional, Dict
+from typing import Callable, Any, Optional, Set
 from datetime import datetime
 
 import config
@@ -13,91 +20,89 @@ logger = logging.getLogger(__name__)
 
 class RequestQueue:
     """
-    Queue for managing API requests with rate limiting.
-    Ensures minimum delay between requests.
+    Queue for managing requests to our local stats API.
+
+    Dispatch is paced by a hard floor (min_dispatch_interval): two requests
+    can never be dispatched closer together than that, enforced by a lock in
+    the processor loop, independent of how long any individual request takes
+    to complete. Actual execution is capped separately by max_concurrent via
+    a semaphore, so a slow request can't stall the whole queue behind it.
     """
-    
-    def __init__(self, min_delay: float = 7.0):
+
+    def __init__(self, min_dispatch_interval: float, max_concurrent: int):
         """
-        Initialize request queue.
-        
         Args:
-            min_delay: Minimum delay in seconds between requests (default: 7.0)
+            min_dispatch_interval: Minimum seconds between successive dispatches
+            max_concurrent: Maximum requests in flight at once (set to 1 to
+                fully serialize execution again, e.g. as a rollback lever)
         """
-        self.min_delay = min_delay
+        self.min_dispatch_interval = min_dispatch_interval
+        self.max_concurrent = max_concurrent
         self.queue = asyncio.Queue()
         self.is_processing = False
-        self.last_request_time: Optional[float] = None
+        self.last_dispatch_time: Optional[float] = None
         self.lock = asyncio.Lock()
+        self.semaphore = asyncio.Semaphore(max_concurrent)
         self._processor_task: Optional[asyncio.Task] = None
-    
+        self._in_flight: Set[asyncio.Task] = set()
+
     async def add_request(self, request_func: Callable, *args, **kwargs) -> Any:
         """
         Add a request to the queue and wait for its result.
-        
+
         Args:
             request_func: Async function to call
             *args: Positional arguments for the function
             **kwargs: Keyword arguments for the function
-            
+
         Returns:
             Result of the request function
         """
-        # Create a future to wait for the result
         future = asyncio.Future()
-        
-        # Add request to queue
+
         await self.queue.put({
             'func': request_func,
             'args': args,
             'kwargs': kwargs,
             'future': future
         })
-        
-        # Start processor if not already running
+
         if not self.is_processing:
             self._start_processor()
-        
-        # Wait for result
+
         return await future
-    
+
     def _start_processor(self):
         """Start the queue processor task"""
         if self._processor_task is None or self._processor_task.done():
             self.is_processing = True
             self._processor_task = asyncio.create_task(self._process_queue())
-            logger.info(f"🚀 Started request queue processor (delay: {self.min_delay}s)")
-    
+            logger.info(
+                f"🚀 Started request queue processor "
+                f"(dispatch interval: {self.min_dispatch_interval}s, "
+                f"max concurrent: {self.max_concurrent})"
+            )
+
     async def _process_queue(self):
-        """Process requests from the queue with rate limiting"""
+        """Dequeue requests and dispatch them at a paced rate, without waiting for completion"""
         logger.info("📋 Request queue processor started")
-        
+
         while True:
             try:
-                # Get next request from queue (wait indefinitely)
                 request_item = await self.queue.get()
-                
-                # Wait if needed to maintain minimum delay
-                await self._wait_if_needed()
-                
-                # Execute the request
-                func = request_item['func']
-                args = request_item['args']
-                kwargs = request_item['kwargs']
-                future = request_item['future']
-                
-                try:
-                    logger.debug(f"🔄 Executing request: {func.__name__}")
-                    result = await func(*args, **kwargs)
-                    future.set_result(result)
-                    logger.debug(f"✅ Request completed: {func.__name__}")
-                except Exception as e:
-                    logger.error(f"❌ Request failed: {func.__name__}: {e}")
-                    future.set_exception(e)
-                
-                # Mark task as done
+
+                # Hard floor: never dispatch two requests closer than
+                # min_dispatch_interval apart, regardless of how long the
+                # previous one takes to finish (that's what self.semaphore
+                # bounds separately, not this gate).
+                await self._wait_for_dispatch_slot()
+
+                task = asyncio.create_task(self._execute(request_item))
+                self._in_flight.add(task)
+                task.add_done_callback(self._in_flight.discard)
+
                 self.queue.task_done()
-                
+
             except asyncio.CancelledError:
                 logger.info("🛑 Request queue processor cancelled")
                 break
@@ -105,32 +110,56 @@ class RequestQueue:
                 logger.error(f"❌ Error in request queue processor: {e}")
                 import traceback
                 logger.error(traceback.format_exc())
-                # Continue processing
                 await asyncio.sleep(1)
-    
-    async def _wait_if_needed(self):
-        """Wait if necessary to maintain minimum delay between requests"""
+
+    async def _execute(self, request_item: dict):
+        """Run one queued request, bounded by the concurrency semaphore"""
+        func = request_item['func']
+        args = request_item['args']
+        kwargs = request_item['kwargs']
+        future = request_item['future']
+
+        async with self.semaphore:
+            try:
+                logger.debug(f"🔄 Executing request: {func.__name__}")
+                result = await func(*args, **kwargs)
+                if not future.done():
+                    future.set_result(result)
+                logger.debug(f"✅ Request completed: {func.__name__}")
+            except Exception as e:
+                logger.error(f"❌ Request failed: {func.__name__}: {e}")
+                if not future.done():
+                    future.set_exception(e)
+
+    async def _wait_for_dispatch_slot(self):
+        """Wait if necessary to maintain minimum delay between dispatches"""
         async with self.lock:
             now = datetime.now().timestamp()
-            
-            if self.last_request_time is not None:
-                elapsed = now - self.last_request_time
-                if elapsed < self.min_delay:
-                    wait_time = self.min_delay - elapsed
-                    logger.debug(f"⏳ Rate limiter: waiting {wait_time:.2f} seconds")
+
+            if self.last_dispatch_time is not None:
+                elapsed = now - self.last_dispatch_time
+                if elapsed < self.min_dispatch_interval:
+                    wait_time = self.min_dispatch_interval - elapsed
+                    logger.debug(f"⏳ Dispatch pacing: waiting {wait_time:.2f} seconds")
                     await asyncio.sleep(wait_time)
                     now = datetime.now().timestamp()
-            
-            self.last_request_time = now
-    
+
+            self.last_dispatch_time = now
+
     async def stop(self):
-        """Stop the queue processor"""
+        """Stop the queue processor and any in-flight requests"""
         if self._processor_task and not self._processor_task.done():
             self._processor_task.cancel()
             try:
                 await self._processor_task
             except asyncio.CancelledError:
                 pass
+
+        for task in list(self._in_flight):
+            task.cancel()
+        if self._in_flight:
+            await asyncio.gather(*self._in_flight, return_exceptions=True)
+
         self.is_processing = False
         logger.info("🛑 Request queue processor stopped")
 
@@ -141,6 +170,8 @@ def get_request_queue() -> RequestQueue:
     """Get the global request queue instance"""
     global _request_queue
     if _request_queue is None:
-        _request_queue = RequestQueue(min_delay=config.LICHESS_REQUEST_QUEUE_MIN_DELAY)
+        _request_queue = RequestQueue(
+            min_dispatch_interval=config.LICHESS_REQUEST_QUEUE_MIN_DISPATCH_INTERVAL,
+            max_concurrent=config.LICHESS_REQUEST_QUEUE_MAX_CONCURRENT,
+        )
     return _request_queue
-