Initial scaffold: .gitignore, config, requirements, data seeds, directory structure

utils/http_limiter.py

@@ -0,0 +1,45 @@
+"""Token-bucket rate limiter for HTTP requests."""
+import time
+import random
+import threading
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class HTTPRateLimiter:
+    """Token-bucket rate limiter with jitter for polite crawling."""
+
+    def __init__(self, rate: float = 1.0, jitter: float = 0.5):
+        """
+        Args:
+            rate: Maximum requests per second
+            jitter: Random jitter range in seconds (±jitter/2)
+        """
+        self.rate = rate
+        self.jitter = jitter
+        self.min_interval = 1.0 / rate if rate > 0 else 0
+        self._last_request = 0.0
+        self._lock = threading.Lock()
+
+    def wait(self):
+        """Block until the next request is allowed."""
+        with self._lock:
+            now = time.monotonic()
+            elapsed = now - self._last_request
+            wait_time = self.min_interval - elapsed
+
+            if self.jitter > 0:
+                wait_time += random.uniform(-self.jitter / 2, self.jitter / 2)
+
+            if wait_time > 0:
+                logger.debug("Rate limiting: waiting %.2fs", wait_time)
+                time.sleep(wait_time)
+
+            self._last_request = time.monotonic()
+
+    def update_rate(self, rate: float):
+        """Update the rate limit (e.g., from profile config)."""
+        with self._lock:
+            self.rate = rate
+            self.min_interval = 1.0 / rate if rate > 0 else 0