n0mad1k
26 lines
540 B
YAML
26 lines
540 B
YAML
name: cia_rdp
|
|
description: "CIA Reading Room - Declassified documents (tradecraft, HUMINT, operations)"
|
|
base_url: "https://www.cia.gov/readingroom/collection/what-was-the-cia-doing"
|
|
domain_allowlist:
|
|
- www.cia.gov
|
|
- cia.gov
|
|
crawl_rules:
|
|
max_depth: 3
|
|
max_documents: 500
|
|
follow_links: true
|
|
url_patterns:
|
|
- "/readingroom/*"
|
|
- "*.pdf"
|
|
exclude_patterns:
|
|
- "*.css"
|
|
- "*.js"
|
|
- "*.ico"
|
|
- "*.png"
|
|
- "*.jpg"
|
|
content_types:
|
|
- text/html
|
|
- application/pdf
|
|
expected_min_count: 50
|
|
rate_limit: 0.5
|
|
mirrors: []
|