# Example source profile — copy and customize for your own sources
# Place in collectors/profiles/ or pass with --profile flag
name: my_source
description: "Description of the document source"
base_url: "https://example.com/documents/"
domain_allowlist:
  - example.com
  - www.example.com
crawl_rules:
  max_depth: 2
  max_documents: 500
  follow_links: true
  url_patterns:
    - "/documents/*"
    - "*.pdf"
  exclude_patterns:
    - "*.css"
    - "*.js"
    - "*.ico"
    - "*.png"
    - "*.jpg"
content_types:
  - text/html
  - application/pdf
expected_min_count: 100
rate_limit: 1.0
mirrors: []