Add collector engine: base, custom source, profile loader, wikileaks helpers

collectors/__init__.py

@@ -0,0 +1,3 @@
+from .base import BaseCollector
+from .custom import CustomSource
+from .profile_loader import load_profile, list_profiles

collectors/base.py

@@ -0,0 +1,292 @@
+"""Abstract base collector for Mosaic document acquisition."""
+import os
+import tempfile
+import hashlib
+import logging
+from abc import ABC, abstractmethod
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import httpx
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+
+from storage.db import MosaicDB
+from storage.models import Document
+from utils.http_limiter import HTTPRateLimiter
+from utils.cache import DownloadCache
+from utils.sanitize import sanitize_filename, validate_content_type
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCollector(ABC):
+    """Abstract base for all document collectors."""
+
+    def __init__(self, db: MosaicDB, cache: DownloadCache,
+                 rate_limiter: Optional[HTTPRateLimiter] = None,
+                 config: Optional[dict] = None):
+        self.db = db
+        self.cache = cache
+        self.rate_limiter = rate_limiter or HTTPRateLimiter()
+        self.config = config or {}
+        self.user_agent = self.config.get('user_agent', 'Mozilla/5.0 (compatible; research-crawler/1.0)')
+        self.timeout = int(self.config.get('timeout', 30))
+        self.max_retries = int(self.config.get('max_retries', 3))
+
+    @abstractmethod
+    def discover(self, since: Optional[str] = None,
+                 until: Optional[str] = None,
+                 limit: Optional[int] = None) -> list[dict]:
+        """Discover document URLs from the source.
+
+        Returns list of dicts: {'url': str, 'title': str, 'date': str|None, 'doc_type': str}
+        """
+        pass
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """Return the source name."""
+        pass
+
+    def expected_count(self) -> int:
+        """Minimum expected document count for validation."""
+        return 0
+
+    def fetch(self, url: str, collection: str) -> Optional[Document]:
+        """Download a URL with atomic writes and hash verification.
+
+        Returns Document if successful, None if failed.
+        """
+        # Check if already cached
+        existing = self.db.get_document_by_url(url)
+        if existing and existing.status == 'CACHED':
+            cached_path = self.cache.get_cached_path(
+                existing.content_hash, collection,
+                self._url_extension(url)
+            )
+            if cached_path:
+                logger.debug("Already cached: %s", url)
+                return existing
+
+        # Rate limit
+        self.rate_limiter.wait()
+
+        # Download to temp file
+        try:
+            with httpx.Client(timeout=self.timeout, follow_redirects=True) as client:
+                response = client.get(url, headers={'User-Agent': self.user_agent})
+                response.raise_for_status()
+
+                content = response.content
+                content_hash = hashlib.sha256(content).hexdigest()
+                content_length = len(content)
+                etag = response.headers.get('etag')
+                last_modified = response.headers.get('last-modified')
+                content_type = response.headers.get('content-type', '').split(';')[0].strip()
+
+                # Write to temp file
+                ext = self._url_extension(url) or self._mime_to_ext(content_type)
+                tmp_fd, tmp_path = tempfile.mkstemp(suffix=f'.{ext}.tmp', dir=str(self.cache.base_dir))
+                try:
+                    os.write(tmp_fd, content)
+                    os.close(tmp_fd)
+
+                    # Validate content type by magic bytes
+                    try:
+                        detected_type = validate_content_type(tmp_path)
+                    except ValueError:
+                        detected_type = content_type
+
+                    # Atomic store
+                    final_path = self.cache.store_atomic(
+                        tmp_path, content_hash, collection, ext, content_length
+                    )
+                except Exception:
+                    # Clean up temp file on failure
+                    os.close(tmp_fd) if not os.path.exists(tmp_path) else None
+                    if os.path.exists(tmp_path):
+                        os.unlink(tmp_path)
+                    raise
+
+                # Build document
+                doc_type = self._detect_doc_type(detected_type)
+                doc = Document(
+                    id=content_hash,
+                    source=collection,
+                    source_url=url,
+                    doc_type=doc_type,
+                    title=self._extract_title(url),
+                    raw_path=final_path,
+                    content_hash=content_hash,
+                    fetch_date=datetime.utcnow().isoformat(),
+                    etag=etag,
+                    last_modified=last_modified,
+                    char_count=len(content.decode('utf-8', errors='replace')),
+                    status='CACHED',
+                    parse_status='PENDING',
+                )
+
+                self.db.insert_document(doc)
+                return doc
+
+        except httpx.HTTPStatusError as e:
+            logger.error("HTTP %d for %s: %s", e.response.status_code, url, e)
+        except httpx.RequestError as e:
+            logger.error("Request error for %s: %s", url, e)
+        except Exception as e:
+            logger.error("Download failed for %s: %s", url, e)
+
+        # Record failure
+        if existing:
+            self.db.update_document_status(existing.id, status='DOWNLOAD_FAILED')
+        return None
+
+    def check_update(self, url: str) -> bool:
+        """Check if a cached URL has been updated (HTTP HEAD with ETag/Last-Modified).
+
+        Returns True if the document has changed and should be re-downloaded.
+        """
+        existing = self.db.get_document_by_url(url)
+        if not existing:
+            return True
+
+        try:
+            self.rate_limiter.wait()
+            with httpx.Client(timeout=self.timeout, follow_redirects=True) as client:
+                response = client.head(url, headers={'User-Agent': self.user_agent})
+                response.raise_for_status()
+
+                new_etag = response.headers.get('etag')
+                new_modified = response.headers.get('last-modified')
+
+                if existing.etag and new_etag:
+                    return existing.etag != new_etag
+                if existing.last_modified and new_modified:
+                    return existing.last_modified != new_modified
+
+                # Can't determine — assume changed
+                return True
+
+        except Exception as e:
+            logger.warning("HEAD check failed for %s: %s", url, e)
+            return False
+
+    def collect(self, since: Optional[str] = None, until: Optional[str] = None,
+                limit: Optional[int] = None, update: bool = False,
+                retry_failed: bool = False) -> dict:
+        """Main collection entry point with progress tracking.
+
+        Returns dict with counts: {'discovered', 'downloaded', 'cached', 'failed', 'updated'}
+        """
+        name = self.get_name()
+        stats = {'discovered': 0, 'downloaded': 0, 'cached': 0, 'failed': 0, 'updated': 0}
+
+        # Check disk space
+        if not self.cache.check_disk_space(required_mb=100):
+            logger.error("Insufficient disk space for collection")
+            return stats
+
+        # Discover URLs
+        urls = self.discover(since=since, until=until, limit=limit)
+        stats['discovered'] = len(urls)
+
+        # Validate against expected count
+        expected = self.expected_count()
+        if expected > 0 and len(urls) < expected * 0.1:
+            logger.warning(
+                "Discovered only %d URLs (expected >%d). "
+                "Site structure may have changed.",
+                len(urls), int(expected * 0.1)
+            )
+
+        # Filter for update mode
+        if update:
+            urls_to_process = []
+            for item in urls:
+                if self.check_update(item['url']):
+                    urls_to_process.append(item)
+                    stats['updated'] += 1
+            urls = urls_to_process
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            transient=True,
+        ) as progress:
+            task = progress.add_task(f"Collecting {name}", total=len(urls))
+
+            for item in urls:
+                url = item['url']
+
+                # Skip already cached (unless update mode)
+                existing = self.db.get_document_by_url(url)
+                if existing and existing.status == 'CACHED' and not update:
+                    stats['cached'] += 1
+                    progress.advance(task)
+                    continue
+
+                # Skip failed unless retry
+                if existing and existing.status == 'DOWNLOAD_FAILED' and not retry_failed:
+                    stats['failed'] += 1
+                    progress.advance(task)
+                    continue
+
+                doc = self.fetch(url, name)
+                if doc:
+                    # Update with discovered metadata
+                    if item.get('title'):
+                        doc.title = item['title']
+                    if item.get('date'):
+                        doc.date = item['date']
+                    self.db.insert_document(doc)
+                    stats['downloaded'] += 1
+                else:
+                    stats['failed'] += 1
+
+                progress.advance(task)
+
+        return stats
+
+    def _url_extension(self, url: str) -> str:
+        """Extract file extension from URL."""
+        from urllib.parse import urlparse
+        path = urlparse(url).path
+        ext = Path(path).suffix.lstrip('.')
+        if ext and len(ext) <= 10:
+            return sanitize_filename(ext).lower()
+        return 'html'
+
+    def _mime_to_ext(self, mime: str) -> str:
+        """Convert MIME type to file extension."""
+        mapping = {
+            'text/html': 'html',
+            'application/pdf': 'pdf',
+            'text/plain': 'txt',
+            'message/rfc822': 'eml',
+            'application/json': 'json',
+        }
+        return mapping.get(mime, 'bin')
+
+    def _detect_doc_type(self, content_type: str) -> str:
+        """Map content type to our doc_type enum."""
+        if 'html' in content_type:
+            return 'html'
+        elif 'pdf' in content_type:
+            return 'pdf'
+        elif 'email' in content_type or 'rfc822' in content_type:
+            return 'email'
+        elif 'text' in content_type:
+            return 'text'
+        return 'html'  # default for web pages
+
+    def _extract_title(self, url: str) -> str:
+        """Extract a title from URL path."""
+        from urllib.parse import urlparse, unquote
+        path = urlparse(url).path
+        parts = [p for p in path.split('/') if p]
+        if parts:
+            return unquote(parts[-1]).replace('-', ' ').replace('_', ' ')
+        return url

collectors/custom.py

@@ -0,0 +1,217 @@
+"""Universal source collector configured by YAML profiles."""
+import re
+import logging
+from urllib.parse import urlparse, urljoin, unquote
+from typing import Optional
+from datetime import datetime
+
+import httpx
+from bs4 import BeautifulSoup
+
+from .base import BaseCollector
+from storage.db import MosaicDB
+from utils.http_limiter import HTTPRateLimiter
+from utils.cache import DownloadCache
+
+logger = logging.getLogger(__name__)
+
+
+class CustomSource(BaseCollector):
+    """Universal collector that crawls any source based on profile configuration."""
+
+    def __init__(self, profile: dict, db: MosaicDB, cache: DownloadCache,
+                 rate_limiter: Optional[HTTPRateLimiter] = None,
+                 config: Optional[dict] = None):
+        # Set rate from profile
+        rate = profile.get('rate_limit', 1.0)
+        if rate_limiter is None:
+            rate_limiter = HTTPRateLimiter(rate=rate, jitter=0.5)
+
+        super().__init__(db, cache, rate_limiter, config)
+        self.profile = profile
+        self.name = profile['name']
+        self.base_url = profile['base_url']
+        self.domain_allowlist = set(profile.get('domain_allowlist', []))
+        self.crawl_rules = profile.get('crawl_rules', {})
+        self.content_types = set(profile.get('content_types', ['text/html', 'application/pdf']))
+        self.mirrors = profile.get('mirrors', [])
+
+        # Compile URL patterns
+        self.url_patterns = [
+            re.compile(self._glob_to_regex(p))
+            for p in self.crawl_rules.get('url_patterns', [])
+        ]
+        self.exclude_patterns = [
+            re.compile(self._glob_to_regex(p))
+            for p in self.crawl_rules.get('exclude_patterns', [])
+        ]
+
+        # Import wikileaks helpers if applicable
+        self._wikileaks_helpers = None
+        if any(d.endswith('wikileaks.org') for d in self.domain_allowlist):
+            try:
+                from . import wikileaks
+                self._wikileaks_helpers = wikileaks
+                logger.debug("WikiLeaks helpers loaded for %s", self.name)
+            except ImportError:
+                pass
+
+    def get_name(self) -> str:
+        return self.name
+
+    def expected_count(self) -> int:
+        return self.profile.get('expected_min_count', 0)
+
+    def discover(self, since: Optional[str] = None,
+                 until: Optional[str] = None,
+                 limit: Optional[int] = None) -> list[dict]:
+        """Crawl source to discover document URLs."""
+        max_depth = self.crawl_rules.get('max_depth', 2)
+        max_docs = limit or self.crawl_rules.get('max_documents', 1000)
+
+        discovered = []
+        visited = set()
+        queue = [(self.base_url, 0)]
+
+        while queue and len(discovered) < max_docs:
+            url, depth = queue.pop(0)
+
+            if url in visited:
+                continue
+            visited.add(url)
+
+            if depth > max_depth:
+                continue
+
+            # Domain check
+            if not self._is_allowed_domain(url):
+                continue
+
+            # Exclude check
+            if self._is_excluded(url):
+                continue
+
+            logger.debug("Crawling: %s (depth %d)", url, depth)
+            self.rate_limiter.wait()
+
+            try:
+                with httpx.Client(timeout=self.timeout, follow_redirects=True) as client:
+                    response = client.get(url, headers={'User-Agent': self.user_agent})
+                    response.raise_for_status()
+                    content_type = response.headers.get('content-type', '').split(';')[0].strip()
+
+                    # If it's a document (not just a page to crawl), add it
+                    if self._matches_url_pattern(url) or self._is_document_type(content_type):
+                        item = {
+                            'url': url,
+                            'title': self._extract_title_from_url(url),
+                            'date': None,
+                            'doc_type': self._content_type_to_doc_type(content_type),
+                        }
+
+                        # Date filtering
+                        if since and item.get('date') and item['date'] < since:
+                            continue
+                        if until and item.get('date') and item['date'] > until:
+                            continue
+
+                        discovered.append(item)
+
+                    # Extract links if HTML and should follow links
+                    if 'html' in content_type and self.crawl_rules.get('follow_links', True):
+                        links = self._extract_links(response.text, url)
+
+                        # Use wikileaks helpers if available
+                        if self._wikileaks_helpers:
+                            extra = self._wikileaks_helpers.extract_extra_links(
+                                response.text, url, self.name
+                            )
+                            links.extend(extra)
+
+                        for link in links:
+                            if link not in visited and len(discovered) < max_docs:
+                                queue.append((link, depth + 1))
+
+            except httpx.HTTPStatusError as e:
+                logger.warning("HTTP %d for %s", e.response.status_code, url)
+            except httpx.RequestError as e:
+                logger.warning("Request error for %s: %s", url, e)
+            except Exception as e:
+                logger.warning("Error crawling %s: %s", url, e)
+
+        logger.info("Discovered %d documents from %s", len(discovered), self.name)
+        return discovered[:max_docs]
+
+    def _is_allowed_domain(self, url: str) -> bool:
+        """Check if URL domain is in the allowlist."""
+        if not self.domain_allowlist:
+            return True
+        parsed = urlparse(url)
+        return parsed.netloc in self.domain_allowlist
+
+    def _is_excluded(self, url: str) -> bool:
+        """Check if URL matches any exclude pattern."""
+        for pattern in self.exclude_patterns:
+            if pattern.search(url):
+                return True
+        return False
+
+    def _matches_url_pattern(self, url: str) -> bool:
+        """Check if URL matches any include pattern."""
+        if not self.url_patterns:
+            return True
+        for pattern in self.url_patterns:
+            if pattern.search(url):
+                return True
+        return False
+
+    def _is_document_type(self, content_type: str) -> bool:
+        """Check if content type is one we want to collect."""
+        return content_type in self.content_types
+
+    def _extract_links(self, html: str, base_url: str) -> list[str]:
+        """Extract absolute URLs from HTML page."""
+        links = []
+        try:
+            soup = BeautifulSoup(html, 'html.parser')
+            for tag in soup.find_all('a', href=True):
+                href = tag['href']
+                # Skip fragments, javascript, mailto
+                if href.startswith(('#', 'javascript:', 'mailto:')):
+                    continue
+                absolute = urljoin(base_url, href)
+                # Strip fragments
+                absolute = absolute.split('#')[0]
+                if absolute and self._is_allowed_domain(absolute):
+                    links.append(absolute)
+        except Exception as e:
+            logger.warning("Link extraction failed for %s: %s", base_url, e)
+        return links
+
+    def _extract_title_from_url(self, url: str) -> str:
+        """Extract a title from URL path."""
+        path = urlparse(url).path
+        parts = [p for p in path.split('/') if p]
+        if parts:
+            return unquote(parts[-1]).replace('-', ' ').replace('_', ' ')
+        return url
+
+    def _content_type_to_doc_type(self, content_type: str) -> str:
+        """Map content type to doc_type."""
+        if 'html' in content_type:
+            return 'html'
+        elif 'pdf' in content_type:
+            return 'pdf'
+        elif 'text' in content_type:
+            return 'text'
+        return 'html'
+
+    @staticmethod
+    def _glob_to_regex(pattern: str) -> str:
+        """Convert glob pattern to regex."""
+        # Escape regex special chars except * and ?
+        pattern = re.escape(pattern)
+        # Convert glob wildcards
+        pattern = pattern.replace(r'\*', '.*')
+        pattern = pattern.replace(r'\?', '.')
+        return pattern

collectors/profile_loader.py

@@ -0,0 +1,129 @@
+"""Load YAML source profiles for Mosaic collectors."""
+import logging
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+PROFILES_DIR = Path(__file__).parent / 'profiles'
+
+REQUIRED_FIELDS = ['name', 'base_url', 'domain_allowlist']
+DEFAULT_CRAWL_RULES = {
+    'max_depth': 2,
+    'max_documents': 1000,
+    'follow_links': True,
+    'url_patterns': [],
+    'exclude_patterns': ['*.css', '*.js', '*.ico', '*.png', '*.jpg', '*.gif'],
+}
+
+
+def load_profile(name_or_path: str) -> dict:
+    """Load a source profile by name or file path.
+
+    Args:
+        name_or_path: Profile name (e.g., 'vault7') or path to YAML file
+
+    Returns:
+        Profile dict with all fields populated (defaults filled in)
+
+    Raises:
+        FileNotFoundError: If profile not found
+        ValueError: If profile missing required fields
+    """
+    path = Path(name_or_path)
+
+    # Check if it's a direct path
+    if path.exists() and path.suffix in ('.yaml', '.yml'):
+        profile_path = path
+    else:
+        # Look in profiles directory
+        profile_path = PROFILES_DIR / f"{name_or_path}.yaml"
+        if not profile_path.exists():
+            profile_path = PROFILES_DIR / f"{name_or_path}.yml"
+
+    if not profile_path.exists():
+        raise FileNotFoundError(f"Profile not found: {name_or_path}")
+
+    with open(profile_path, 'r') as f:
+        profile = yaml.safe_load(f)
+
+    if not profile:
+        raise ValueError(f"Empty profile: {profile_path}")
+
+    # Validate required fields
+    for field in REQUIRED_FIELDS:
+        if field not in profile:
+            raise ValueError(f"Profile missing required field: {field}")
+
+    # Fill defaults
+    profile.setdefault('description', '')
+    profile.setdefault('content_types', ['text/html', 'application/pdf'])
+    profile.setdefault('expected_min_count', 0)
+    profile.setdefault('rate_limit', 1.0)
+    profile.setdefault('mirrors', [])
+
+    crawl_rules = profile.get('crawl_rules', {})
+    for key, default in DEFAULT_CRAWL_RULES.items():
+        crawl_rules.setdefault(key, default)
+    profile['crawl_rules'] = crawl_rules
+
+    return profile
+
+
+def create_adhoc_profile(url: str, name: str) -> dict:
+    """Create an ad-hoc profile for a URL.
+
+    Args:
+        url: Base URL to crawl
+        name: Name for this source
+
+    Returns:
+        Profile dict
+    """
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    domain = parsed.netloc
+
+    return {
+        'name': name,
+        'description': f"Ad-hoc source: {url}",
+        'base_url': url,
+        'domain_allowlist': [domain],
+        'crawl_rules': dict(DEFAULT_CRAWL_RULES),
+        'content_types': ['text/html', 'application/pdf'],
+        'expected_min_count': 0,
+        'rate_limit': 1.0,
+        'mirrors': [],
+    }
+
+
+def list_profiles() -> list[dict]:
+    """List all available source profiles.
+
+    Returns:
+        List of dicts with 'name', 'description', 'path' for each profile
+    """
+    profiles = []
+
+    if not PROFILES_DIR.exists():
+        return profiles
+
+    for path in sorted(PROFILES_DIR.glob('*.yaml')):
+        if path.stem == 'example':
+            continue
+        try:
+            with open(path, 'r') as f:
+                data = yaml.safe_load(f)
+            if data:
+                profiles.append({
+                    'name': data.get('name', path.stem),
+                    'description': data.get('description', ''),
+                    'path': str(path),
+                    'expected_count': data.get('expected_min_count', 0),
+                })
+        except Exception as e:
+            logger.warning("Failed to load profile %s: %s", path, e)
+
+    return profiles

collectors/wikileaks.py

@@ -0,0 +1,142 @@
+"""WikiLeaks-specific crawl helpers for Mosaic collectors."""
+import re
+import logging
+from urllib.parse import urljoin
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def extract_extra_links(html: str, base_url: str, source_name: str) -> list[str]:
+    """Extract additional links using WikiLeaks-specific patterns.
+
+    Different WikiLeaks collections have different page structures.
+    This function handles the quirks of each.
+    """
+    links = []
+
+    if source_name == 'vault7':
+        links.extend(_vault7_links(html, base_url))
+    elif source_name == 'vault8':
+        links.extend(_vault8_links(html, base_url))
+    elif source_name == 'cablegate':
+        links.extend(_cablegate_links(html, base_url))
+    elif source_name == 'spyfiles':
+        links.extend(_spyfiles_links(html, base_url))
+    elif source_name == 'emails':
+        links.extend(_email_links(html, base_url))
+
+    return links
+
+
+def _vault7_links(html: str, base_url: str) -> list[str]:
+    """Extract Vault 7 project page links.
+
+    Vault 7 has a main index with project names that link to sub-pages.
+    Each project page has document links.
+    """
+    links = []
+    # Match project links like /vault7/#Dark Matter
+    for match in re.finditer(r'href=["\'](/vault7/[^"\']+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    # Match document/file links
+    for match in re.finditer(r'href=["\']([^"\']*(?:\.pdf|\.zip|\.tar\.gz)[^"\']*)["\']', html, re.IGNORECASE):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    return links
+
+
+def _vault8_links(html: str, base_url: str) -> list[str]:
+    """Extract Vault 8 source code repository links."""
+    links = []
+    for match in re.finditer(r'href=["\'](/vault8/[^"\']+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+    return links
+
+
+def _cablegate_links(html: str, base_url: str) -> list[str]:
+    """Extract Cablegate cable links with pagination support.
+
+    Cablegate uses /plusd/cables/ with date-based navigation.
+    Individual cables are at /plusd/cables/CABLE_ID
+    """
+    links = []
+
+    # Individual cable links
+    for match in re.finditer(r'href=["\'](/plusd/cables/[0-9A-Z]+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    # Pagination links
+    for match in re.finditer(r'href=["\'](/plusd/[^"\']*(?:page|offset|start)=[^"\']+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    # Date-based navigation
+    for match in re.finditer(r'href=["\'](/plusd/cables/\?[^"\']+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    return links
+
+
+def _spyfiles_links(html: str, base_url: str) -> list[str]:
+    """Extract Spy Files document links."""
+    links = []
+    for match in re.finditer(r'href=["\'](/spyfiles/[^"\']+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+    return links
+
+
+def _email_links(html: str, base_url: str) -> list[str]:
+    """Extract email dump links with ID-based navigation."""
+    links = []
+
+    # Email ID links
+    for match in re.finditer(r'href=["\']([^"\']*emailid/\d+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    # Pagination
+    for match in re.finditer(r'href=["\']([^"\']*(?:page|offset)=\d+)["\']', html):
+        link = urljoin(base_url, match.group(1))
+        links.append(link)
+
+    return links
+
+
+def parse_cable_date(cable_id: str) -> Optional[str]:
+    """Extract date from cable ID format.
+
+    Cable IDs often start with a two-digit year and month:
+    e.g., 07PARIS1234 → 2007 (approximate)
+    """
+    match = re.match(r'^(\d{2})[A-Z]', cable_id)
+    if match:
+        year = int(match.group(1))
+        # WikiLeaks cables span roughly 1966-2010
+        if year >= 66:
+            return f"19{year:02d}"
+        else:
+            return f"20{year:02d}"
+    return None
+
+
+def detect_wikileaks_source(url: str) -> Optional[str]:
+    """Detect which WikiLeaks collection a URL belongs to."""
+    if '/vault7/' in url:
+        return 'vault7'
+    elif '/vault8/' in url:
+        return 'vault8'
+    elif '/plusd/' in url or '/cablegate/' in url:
+        return 'cablegate'
+    elif '/spyfiles/' in url:
+        return 'spyfiles'
+    elif 'emailid/' in url or '/dnc-emails/' in url or '/podesta-emails/' in url:
+        return 'emails'
+    return None