Fix crawler scoping: BFS stays within source URL prefix, only collect pattern-matched URLs

collectors/custom.py

@@ -91,6 +91,16 @@ class CustomSource(BaseCollector):
             if self._is_excluded(url):
                 continue
 
+            # Scope check: only crawl URLs under the base_url path prefix
+            # This prevents BFS from wandering the entire domain
+            base_path = urlparse(self.base_url).path.rstrip('/')
+            url_path = urlparse(url).path.rstrip('/')
+            if base_path and not url_path.startswith(base_path) and url != self.base_url:
+                # Allow the base URL itself, but skip other paths outside our scope
+                # unless they match a URL pattern (e.g., linked documents)
+                if not self._matches_url_pattern(url):
+                    continue
+
             logger.debug("Crawling: %s (depth %d)", url, depth)
             self.rate_limiter.wait()
 
@@ -100,8 +110,8 @@ class CustomSource(BaseCollector):
                     response.raise_for_status()
                     content_type = response.headers.get('content-type', '').split(';')[0].strip()
 
-                    # If it's a document (not just a page to crawl), add it
-                    if self._matches_url_pattern(url) or self._is_document_type(content_type):
+                    # Only collect URLs that match our patterns or are the right content type
+                    if self._matches_url_pattern(url):
                         item = {
                             'url': url,
                             'title': self._extract_title_from_url(url),