diff --git a/collectors/custom.py b/collectors/custom.py index 58ae20e..c3b2d36 100644 --- a/collectors/custom.py +++ b/collectors/custom.py @@ -91,6 +91,16 @@ class CustomSource(BaseCollector): if self._is_excluded(url): continue + # Scope check: only crawl URLs under the base_url path prefix + # This prevents BFS from wandering the entire domain + base_path = urlparse(self.base_url).path.rstrip('/') + url_path = urlparse(url).path.rstrip('/') + if base_path and not url_path.startswith(base_path) and url != self.base_url: + # Allow the base URL itself, but skip other paths outside our scope + # unless they match a URL pattern (e.g., linked documents) + if not self._matches_url_pattern(url): + continue + logger.debug("Crawling: %s (depth %d)", url, depth) self.rate_limiter.wait() @@ -100,8 +110,8 @@ class CustomSource(BaseCollector): response.raise_for_status() content_type = response.headers.get('content-type', '').split(';')[0].strip() - # If it's a document (not just a page to crawl), add it - if self._matches_url_pattern(url) or self._is_document_type(content_type): + # Only collect URLs that match our patterns or are the right content type + if self._matches_url_pattern(url): item = { 'url': url, 'title': self._extract_title_from_url(url),