n0mad1k
28 lines
531 B
YAML
28 lines
531 B
YAML
name: vault8
|
|
description: "CIA Source Code (Vault 8)"
|
|
base_url: "https://wikileaks.org/vault8/"
|
|
domain_allowlist:
|
|
- wikileaks.org
|
|
- file.wikileaks.org
|
|
crawl_rules:
|
|
max_depth: 3
|
|
max_documents: 1000
|
|
follow_links: true
|
|
url_patterns:
|
|
- "/vault8/#*"
|
|
- "/vault8/document/*"
|
|
exclude_patterns:
|
|
- "*.css"
|
|
- "*.js"
|
|
- "*.ico"
|
|
- "*.png"
|
|
- "*.jpg"
|
|
content_types:
|
|
- text/html
|
|
- application/pdf
|
|
- text/plain
|
|
expected_min_count: 100
|
|
rate_limit: 1.0
|
|
mirrors:
|
|
- "https://file.wikileaks.org/vault8/"
|