Initial scaffold: .gitignore, config, requirements, data seeds, directory structure

2026-03-19 08:34:32 -04:00
commit aa0f35803f
26 changed files with 1908 additions and 0 deletions
@@ -0,0 +1,20 @@
+name: cablegate
+description: "US Diplomatic Cables (Cablegate)"
+base_url: "https://wikileaks.org/plusd/"
+domain_allowlist:
+  - wikileaks.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 300000
+  follow_links: true
+  url_patterns:
+    - "/plusd/cables/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+content_types:
+  - text/html
+expected_min_count: 250000
+rate_limit: 0.5
+mirrors: []