Initial scaffold: .gitignore, config, requirements, data seeds, directory structure

2026-03-19 08:34:32 -04:00
commit aa0f35803f
26 changed files with 1908 additions and 0 deletions
@@ -0,0 +1,20 @@
+name: cablegate
+description: "US Diplomatic Cables (Cablegate)"
+base_url: "https://wikileaks.org/plusd/"
+domain_allowlist:
+  - wikileaks.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 300000
+  follow_links: true
+  url_patterns:
+    - "/plusd/cables/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+content_types:
+  - text/html
+expected_min_count: 250000
+rate_limit: 0.5
+mirrors: []
@@ -0,0 +1,21 @@
+name: emails
+description: "Email Dumps (DNC, Podesta, etc.)"
+base_url: "https://wikileaks.org/dnc-emails/"
+domain_allowlist:
+  - wikileaks.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 50000
+  follow_links: true
+  url_patterns:
+    - "/dnc-emails/emailid/*"
+    - "/podesta-emails/emailid/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+content_types:
+  - text/html
+expected_min_count: 20000
+rate_limit: 0.5
+mirrors: []
@@ -0,0 +1,27 @@
+# Example source profile — copy and customize for your own sources
+# Place in collectors/profiles/ or pass with --profile flag
+name: my_source
+description: "Description of the document source"
+base_url: "https://example.com/documents/"
+domain_allowlist:
+  - example.com
+  - www.example.com
+crawl_rules:
+  max_depth: 2
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/documents/*"
+    - "*.pdf"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.png"
+    - "*.jpg"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 100
+rate_limit: 1.0
+mirrors: []
@@ -0,0 +1,23 @@
+name: spyfiles
+description: "Surveillance Industry Documents (Spy Files)"
+base_url: "https://wikileaks.org/spyfiles/"
+domain_allowlist:
+  - wikileaks.org
+  - file.wikileaks.org
+crawl_rules:
+  max_depth: 3
+  max_documents: 2000
+  follow_links: true
+  url_patterns:
+    - "/spyfiles/document/*"
+    - "/spyfiles/files/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 200
+rate_limit: 1.0
+mirrors: []
@@ -0,0 +1,27 @@
+name: vault7
+description: "CIA Hacking Tools (Vault 7)"
+base_url: "https://wikileaks.org/vault7/"
+domain_allowlist:
+  - wikileaks.org
+  - file.wikileaks.org
+crawl_rules:
+  max_depth: 3
+  max_documents: 5000
+  follow_links: true
+  url_patterns:
+    - "/vault7/#*"
+    - "/vault7/document/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.png"
+    - "*.jpg"
+    - "*.gif"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 800
+rate_limit: 1.0
+mirrors:
+  - "https://file.wikileaks.org/vault7/"
@@ -0,0 +1,27 @@
+name: vault8
+description: "CIA Source Code (Vault 8)"
+base_url: "https://wikileaks.org/vault8/"
+domain_allowlist:
+  - wikileaks.org
+  - file.wikileaks.org
+crawl_rules:
+  max_depth: 3
+  max_documents: 1000
+  follow_links: true
+  url_patterns:
+    - "/vault8/#*"
+    - "/vault8/document/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.png"
+    - "*.jpg"
+content_types:
+  - text/html
+  - application/pdf
+  - text/plain
+expected_min_count: 100
+rate_limit: 1.0
+mirrors:
+  - "https://file.wikileaks.org/vault8/"