n0mad1k
22 lines
439 B
YAML
22 lines
439 B
YAML
name: emails
|
|
description: "Email Dumps (DNC, Podesta, etc.)"
|
|
base_url: "https://wikileaks.org/dnc-emails/"
|
|
domain_allowlist:
|
|
- wikileaks.org
|
|
crawl_rules:
|
|
max_depth: 2
|
|
max_documents: 50000
|
|
follow_links: true
|
|
url_patterns:
|
|
- "/dnc-emails/emailid/*"
|
|
- "/podesta-emails/emailid/*"
|
|
exclude_patterns:
|
|
- "*.css"
|
|
- "*.js"
|
|
- "*.ico"
|
|
content_types:
|
|
- text/html
|
|
expected_min_count: 20000
|
|
rate_limit: 0.5
|
|
mirrors: []
|