n0mad1k
46 lines
1.4 KiB
Python
46 lines
1.4 KiB
Python
"""Token-bucket rate limiter for HTTP requests."""
|
|
import time
|
|
import random
|
|
import threading
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class HTTPRateLimiter:
|
|
"""Token-bucket rate limiter with jitter for polite crawling."""
|
|
|
|
def __init__(self, rate: float = 1.0, jitter: float = 0.5):
|
|
"""
|
|
Args:
|
|
rate: Maximum requests per second
|
|
jitter: Random jitter range in seconds (±jitter/2)
|
|
"""
|
|
self.rate = rate
|
|
self.jitter = jitter
|
|
self.min_interval = 1.0 / rate if rate > 0 else 0
|
|
self._last_request = 0.0
|
|
self._lock = threading.Lock()
|
|
|
|
def wait(self):
|
|
"""Block until the next request is allowed."""
|
|
with self._lock:
|
|
now = time.monotonic()
|
|
elapsed = now - self._last_request
|
|
wait_time = self.min_interval - elapsed
|
|
|
|
if self.jitter > 0:
|
|
wait_time += random.uniform(-self.jitter / 2, self.jitter / 2)
|
|
|
|
if wait_time > 0:
|
|
logger.debug("Rate limiting: waiting %.2fs", wait_time)
|
|
time.sleep(wait_time)
|
|
|
|
self._last_request = time.monotonic()
|
|
|
|
def update_rate(self, rate: float):
|
|
"""Update the rate limit (e.g., from profile config)."""
|
|
with self._lock:
|
|
self.rate = rate
|
|
self.min_interval = 1.0 / rate if rate > 0 else 0
|