"""Token-bucket rate limiter for HTTP requests.""" import time import random import threading import logging logger = logging.getLogger(__name__) class HTTPRateLimiter: """Token-bucket rate limiter with jitter for polite crawling.""" def __init__(self, rate: float = 1.0, jitter: float = 0.5): """ Args: rate: Maximum requests per second jitter: Random jitter range in seconds (±jitter/2) """ self.rate = rate self.jitter = jitter self.min_interval = 1.0 / rate if rate > 0 else 0 self._last_request = 0.0 self._lock = threading.Lock() def wait(self): """Block until the next request is allowed.""" with self._lock: now = time.monotonic() elapsed = now - self._last_request wait_time = self.min_interval - elapsed if self.jitter > 0: wait_time += random.uniform(-self.jitter / 2, self.jitter / 2) if wait_time > 0: logger.debug("Rate limiting: waiting %.2fs", wait_time) time.sleep(wait_time) self._last_request = time.monotonic() def update_rate(self, rate: float): """Update the rate limit (e.g., from profile config).""" with self._lock: self.rate = rate self.min_interval = 1.0 / rate if rate > 0 else 0