From 7cbc79c229f3886ec29e1deafc37ce2dd13fc962 Mon Sep 17 00:00:00 2001
From: n0mad1k <n0mad1k@proton.me>
Date: Thu, 19 Mar 2026 10:22:02 -0400
Subject: [PATCH] Add source profiles for physical tradecraft: CIA RDP,
 FAS/IRP, Army pubs, Cryptome, Grugq

---
 collectors/profiles/army_pubs.yaml | 24 ++++++++++++++++++++++++
 collectors/profiles/cia_rdp.yaml   | 25 +++++++++++++++++++++++++
 collectors/profiles/cryptome.yaml  | 27 +++++++++++++++++++++++++++
 collectors/profiles/grugq.yaml     | 22 ++++++++++++++++++++++
 collectors/profiles/irp_fas.yaml   | 30 ++++++++++++++++++++++++++++++
 5 files changed, 128 insertions(+)
 create mode 100644 collectors/profiles/army_pubs.yaml
 create mode 100644 collectors/profiles/cia_rdp.yaml
 create mode 100644 collectors/profiles/cryptome.yaml
 create mode 100644 collectors/profiles/grugq.yaml
 create mode 100644 collectors/profiles/irp_fas.yaml

diff --git a/collectors/profiles/army_pubs.yaml b/collectors/profiles/army_pubs.yaml
new file mode 100644
index 0000000..241568f
--- /dev/null
+++ b/collectors/profiles/army_pubs.yaml
@@ -0,0 +1,24 @@
+name: army_pubs
+description: "US Army field manuals - HUMINT, CI, interrogation, surveillance (public)"
+base_url: "https://irp.fas.org/doddir/army/"
+domain_allowlist:
+  - irp.fas.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/doddir/army/*"
+    - "*.pdf"
+    - "*.htm"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.gif"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 50
+rate_limit: 1.0
+mirrors: []
diff --git a/collectors/profiles/cia_rdp.yaml b/collectors/profiles/cia_rdp.yaml
new file mode 100644
index 0000000..026ec48
--- /dev/null
+++ b/collectors/profiles/cia_rdp.yaml
@@ -0,0 +1,25 @@
+name: cia_rdp
+description: "CIA Reading Room - Declassified documents (tradecraft, HUMINT, operations)"
+base_url: "https://www.cia.gov/readingroom/collection/what-was-the-cia-doing"
+domain_allowlist:
+  - www.cia.gov
+  - cia.gov
+crawl_rules:
+  max_depth: 3
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/readingroom/*"
+    - "*.pdf"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.png"
+    - "*.jpg"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 50
+rate_limit: 0.5
+mirrors: []
diff --git a/collectors/profiles/cryptome.yaml b/collectors/profiles/cryptome.yaml
new file mode 100644
index 0000000..0c040b5
--- /dev/null
+++ b/collectors/profiles/cryptome.yaml
@@ -0,0 +1,27 @@
+name: cryptome
+description: "Cryptome - leaked/declassified intelligence documents, tradecraft, surveillance"
+base_url: "https://cryptome.org/"
+domain_allowlist:
+  - cryptome.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/*.htm"
+    - "/*.pdf"
+    - "/*.txt"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.gif"
+    - "*.jpg"
+    - "*.png"
+content_types:
+  - text/html
+  - application/pdf
+  - text/plain
+expected_min_count: 100
+rate_limit: 0.5
+mirrors: []
diff --git a/collectors/profiles/grugq.yaml b/collectors/profiles/grugq.yaml
new file mode 100644
index 0000000..d3e20e1
--- /dev/null
+++ b/collectors/profiles/grugq.yaml
@@ -0,0 +1,22 @@
+name: grugq
+description: "The Grugq's OPSEC and tradecraft writings"
+base_url: "https://grugq.github.io/"
+domain_allowlist:
+  - grugq.github.io
+crawl_rules:
+  max_depth: 2
+  max_documents: 200
+  follow_links: true
+  url_patterns:
+    - "/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.png"
+    - "*.jpg"
+    - "*.ico"
+content_types:
+  - text/html
+expected_min_count: 10
+rate_limit: 1.0
+mirrors: []
diff --git a/collectors/profiles/irp_fas.yaml b/collectors/profiles/irp_fas.yaml
new file mode 100644
index 0000000..5da1870
--- /dev/null
+++ b/collectors/profiles/irp_fas.yaml
@@ -0,0 +1,30 @@
+name: irp_fas
+description: "FAS Intelligence Resource Program - declassified manuals, tradecraft, doctrine"
+base_url: "https://irp.fas.org/cia/"
+domain_allowlist:
+  - irp.fas.org
+crawl_rules:
+  max_depth: 3
+  max_documents: 1000
+  follow_links: true
+  url_patterns:
+    - "/cia/*"
+    - "/nsa/*"
+    - "/dia/*"
+    - "/doddir/*"
+    - "*.pdf"
+    - "*.htm"
+    - "*.html"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.gif"
+    - "*.png"
+    - "*.jpg"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 100
+rate_limit: 1.0
+mirrors: []