Add README, PolyForm NC license, and harden .gitignore for public release

.gitignore

@@ -7,3 +7,21 @@ output/
 __pycache__/
 .venv/
 *.pyc
+.claude/
+
+# Internal dev files — never publish
+CLAUDE.md
+notes.md
+
+# Local corpora / downloaded books — never publish (large, third-party)
+Hacking-Books/
+Hacking-PDF-Books/
+Hacking-Security-Ebooks/
+hacking-books/
+
+# Per-project workflow gate dotfiles (do not commit)
+.active-item.json
+.closeout-required.json
+.workflow-approved
+.workflow-bypass
+.workflow-state.json

LICENSE

@@ -0,0 +1,73 @@
+# PolyForm Noncommercial License 1.0.0
+
+<https://polyformproject.org/licenses/noncommercial/1.0.0>
+
+## Acceptance
+
+In order to get any license under these terms, you must agree to them as both strict obligations and conditions to all your licenses.
+
+## Copyright License
+
+The licensor grants you a copyright license for the software to do everything you might do with the software that would otherwise infringe the licensor's copyright in it for any permitted purpose.  However, you may only distribute the software according to [Distribution License](#distribution-license) and make changes or new works based on the software according to [Changes and New Works License](#changes-and-new-works-license).
+
+## Distribution License
+
+The licensor grants you an additional copyright license to distribute copies of the software.  Your license to distribute covers distributing the software with changes and new works permitted by [Changes and New Works License](#changes-and-new-works-license).
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms or the URL for them above, as well as copies of any plain-text lines beginning with `Required Notice:` that the licensor provided with the software.  For example:
+
+> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
+
+## Changes and New Works License
+
+The licensor grants you an additional copyright license to make changes and new works based on the software for any permitted purpose.
+
+## Patent License
+
+The licensor grants you a patent license for the software that covers patent claims the licensor can license, or becomes able to license, that you would infringe by using the software.
+
+## Noncommercial Purposes
+
+Any noncommercial purpose is a permitted purpose.
+
+## Personal Uses
+
+Personal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, amateur pursuits, or religious observance, without any anticipated commercial application, is use for a permitted purpose.
+
+## Noncommercial Organizations
+
+Use by any charitable organization, educational institution, public research organization, public safety or health organization, environmental protection organization, or government institution is use for a permitted purpose regardless of the source of funding or obligations resulting from the funding.
+
+## Fair Use
+
+You may have "fair use" rights for the software under the law. These terms do not limit them.
+
+## No Other Rights
+
+These terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent the licensor from granting licenses to anyone else.  These terms do not imply any other licenses.
+
+## Patent Defense
+
+If you make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+## Violations
+
+The first time you are notified in writing that you have violated any of these terms, or done anything with the software not covered by your licenses, your licenses can nonetheless continue if you come into full compliance with these terms, and take practical steps to correct past violations, within 32 days of receiving notice.  Otherwise, all your licenses end immediately.
+
+## No Liability
+
+***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.***
+
+## Definitions
+
+The **licensor** is the individual or entity offering these terms, and the **software** is the software the licensor makes available under these terms.
+
+**You** refers to the individual or entity agreeing to these terms.
+
+**Your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization.  **Control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise.  Control can be direct or indirect.
+
+**Your licenses** are all the licenses granted to you for the software under these terms.
+
+**Use** means anything you do with the software requiring one of your licenses.

README.md

@@ -0,0 +1,83 @@
+# mosaic
+
+Intelligence extraction platform for large open-source and declassified document corpora.
+Mosaic collects public leak/disclosure archives (WikiLeaks, Cryptome, FAS/IRP, CIA CREST/RDP,
+Army field manuals, and similar), parses heterogeneous documents, extracts structured
+intelligence (entities, tooling, TTPs, infrastructure, tradecraft), and lets you query and
+synthesize the results — all locally, in SQLite.
+
+> Everything Mosaic ingests is publicly available or previously declassified material.
+> It ships with no corpus and no data — you point it at the sources you are authorized to use.
+
+## Pipeline
+
+```
+collect  →  parse  →  extract  →  analyze  →  query
+```
+
+- **collect** — source-profile-driven crawlers (rate-limited, resumable) into a hash-sharded store
+- **parse** — subprocess-isolated PDF / HTML / email / cable / text parsers with memory + time caps
+- **extract** — Claude-backed extractors: entities, tools, MITRE ATT&CK mapping, infrastructure,
+  surveillance, tradecraft, TTPs
+- **analyze** — BM25/FTS5 context builder, batch + interactive analysis, field-manual generation
+- **query** — full-text search (SQLite FTS5) over everything extracted
+
+## Install
+
+```bash
+git clone https://git.churchofmalware.org/n0mad1k/mosaic.git
+cd mosaic
+python3 -m venv .venv && source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+## Configure
+
+Copy the example config and set your options:
+
+```bash
+cp config/mosaic.conf.example config/mosaic.conf   # if present; otherwise mosaic writes defaults
+```
+
+The Anthropic API key is read from the `ANTHROPIC_API_KEY` environment variable (preferred):
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+A commented `api_key` fallback exists in `config/mosaic.conf`, but the env var takes priority and
+is the recommended path. No key is required for `collect`, `parse`, or `query` — only for the
+Claude-backed `extract`/`analyze` stages.
+
+## Usage
+
+```bash
+python3 mosaic.py                 # interactive Rich menu
+python3 mosaic.py --help          # full CLI
+
+# typical flow
+python3 mosaic.py collect --profile collectors/profiles/cryptome.yaml
+python3 mosaic.py parse
+python3 mosaic.py extract --dry-run     # estimate token cost first
+python3 mosaic.py extract
+python3 mosaic.py query "kerberos delegation"
+```
+
+Source profiles live in `collectors/profiles/`. Copy `example.yaml` to add your own.
+
+## Field manuals
+
+`output/manuals/` contains reference manuals synthesized from public and declassified source
+material (HUMINT, surveillance/counter-surveillance, covert communications, physical access,
+cover & identity, OPSEC, and cyber implants). They are generated artifacts produced by the
+`analyze` stage and are included as worked examples.
+
+## Tests
+
+```bash
+pytest tests/
+```
+
+## License
+
+PolyForm Noncommercial 1.0.0 — see [LICENSE](LICENSE). Commercial licensing on request.

analysis/field_manual.py

@@ -0,0 +1,396 @@
+"""Generate structured Markdown field manual from Mosaic extracted intelligence."""
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+from collections import defaultdict
+
+from storage.db import MosaicDB
+
+logger = logging.getLogger(__name__)
+
+# Domain display order and titles
+TRADECRAFT_DOMAINS = {
+    'humint': 'HUMINT Operations',
+    'surveillance': 'Surveillance',
+    'counter_surveillance': 'Counter-Surveillance',
+    'covert_comms': 'Covert Communications',
+    'physical_access': 'Physical Access & Entry',
+    'escape_evasion': 'Escape & Evasion',
+    'cyber': 'Cyber Operations',
+}
+
+TTP_CATEGORIES = {
+    'reconnaissance': 'Reconnaissance',
+    'resource_development': 'Resource Development',
+    'initial_access': 'Initial Access',
+    'execution': 'Execution',
+    'persistence': 'Persistence',
+    'privilege_escalation': 'Privilege Escalation',
+    'defense_evasion': 'Defense Evasion',
+    'credential_access': 'Credential Access',
+    'discovery': 'Discovery',
+    'lateral_movement': 'Lateral Movement',
+    'collection': 'Collection',
+    'exfiltration': 'Exfiltration',
+    'command_and_control': 'Command & Control',
+    'impact': 'Impact',
+    'counter_surveillance': 'Counter-Surveillance',
+    'covert_comms': 'Covert Communications',
+    'physical_access': 'Physical Access',
+    'humint': 'HUMINT',
+}
+
+TOOL_CAPABILITIES = {
+    'implant': 'Implants & Backdoors',
+    'framework': 'Frameworks',
+    'exploit': 'Exploits',
+    'credential_theft': 'Credential Theft',
+    'collection': 'Collection Tools',
+    'evasion': 'Evasion & Anti-Forensics',
+    'rootkit': 'Rootkits & Covert Storage',
+    'router_exploit': 'Network Device Exploitation',
+    'proxy': 'Proxies & Redirectors',
+    'c2_framework': 'C2 Frameworks',
+    'obfuscation': 'Obfuscation & Attribution',
+    'spyware': 'Spyware',
+    'weapon': 'Weapons Systems',
+    'utility': 'Utilities',
+    'trojan': 'Trojans',
+    'platform': 'Multi-Component Platforms',
+}
+
+
+def generate_field_manual(db: MosaicDB, output_path: str = "output/FIELD_MANUAL.md",
+                          title: str = "Operator's Field Manual") -> str:
+    """Generate the complete field manual from database contents.
+
+    Returns the path to the generated file.
+    """
+    tools = db.get_all_tools()
+    ttps = db.get_all_ttps()
+    tradecraft = db.get_all_tradecraft()
+    entities = db.get_all_entities()
+    infrastructure = db.get_all_infrastructure()
+
+    # Get corpus stats
+    stats = db.get_status_counts()
+    token_stats = db.get_total_tokens()
+
+    sections = []
+
+    # --- HEADER ---
+    sections.append(_header(title, stats, len(tools), len(ttps), len(tradecraft), len(entities), len(infrastructure)))
+
+    # --- TABLE OF CONTENTS ---
+    sections.append(_toc(tools, ttps, tradecraft, entities, infrastructure))
+
+    # --- PART I: TRADECRAFT ---
+    if tradecraft:
+        sections.append("\n---\n\n# Part I: Tradecraft\n")
+        by_domain = defaultdict(list)
+        for tc in tradecraft:
+            by_domain[tc.domain].append(tc)
+
+        for domain_key, domain_title in TRADECRAFT_DOMAINS.items():
+            items = by_domain.get(domain_key, [])
+            if not items:
+                # Check for unlisted domains
+                continue
+            sections.append(f"\n## {domain_title}\n")
+            for tc in sorted(items, key=lambda x: x.method):
+                sections.append(_format_tradecraft(tc))
+
+        # Any domains not in our predefined list
+        for domain_key, items in by_domain.items():
+            if domain_key not in TRADECRAFT_DOMAINS:
+                sections.append(f"\n## {domain_key.replace('_', ' ').title()}\n")
+                for tc in sorted(items, key=lambda x: x.method):
+                    sections.append(_format_tradecraft(tc))
+
+    # --- PART II: TTPs ---
+    if ttps:
+        sections.append("\n---\n\n# Part II: Techniques, Tactics & Procedures\n")
+        by_category = defaultdict(list)
+        for ttp in ttps:
+            by_category[ttp.category].append(ttp)
+
+        for cat_key, cat_title in TTP_CATEGORIES.items():
+            items = by_category.get(cat_key, [])
+            if not items:
+                continue
+            sections.append(f"\n## {cat_title}\n")
+            for ttp in sorted(items, key=lambda x: x.technique):
+                sections.append(_format_ttp(ttp))
+
+        # Unlisted categories
+        for cat_key, items in by_category.items():
+            if cat_key not in TTP_CATEGORIES:
+                sections.append(f"\n## {cat_key.replace('_', ' ').title()}\n")
+                for ttp in sorted(items, key=lambda x: x.technique):
+                    sections.append(_format_ttp(ttp))
+
+    # --- PART III: TOOLS ---
+    if tools:
+        sections.append("\n---\n\n# Part III: Tools & Capabilities\n")
+        by_cap = defaultdict(list)
+        for tool in tools:
+            by_cap[tool.capability].append(tool)
+
+        for cap_key, cap_title in TOOL_CAPABILITIES.items():
+            items = by_cap.get(cap_key, [])
+            if not items:
+                continue
+            sections.append(f"\n## {cap_title}\n")
+            for tool in sorted(items, key=lambda x: x.name):
+                sections.append(_format_tool(tool))
+
+        # Unlisted capabilities
+        for cap_key, items in by_cap.items():
+            if cap_key not in TOOL_CAPABILITIES:
+                sections.append(f"\n## {cap_key.replace('_', ' ').title()}\n")
+                for tool in sorted(items, key=lambda x: x.name):
+                    sections.append(_format_tool(tool))
+
+    # --- PART IV: ENTITIES ---
+    if entities:
+        sections.append("\n---\n\n# Part IV: Entities & Organizations\n")
+        by_type = defaultdict(list)
+        for entity in entities:
+            by_type[entity.type].append(entity)
+
+        for etype in ['unit', 'org', 'program', 'person', 'codename']:
+            items = by_type.get(etype, [])
+            if not items:
+                continue
+            sections.append(f"\n## {etype.title()}s\n")
+            for entity in sorted(items, key=lambda x: x.name):
+                sections.append(_format_entity(entity))
+
+        # Unlisted types
+        for etype, items in by_type.items():
+            if etype not in ['unit', 'org', 'program', 'person', 'codename']:
+                sections.append(f"\n## {etype.title()}\n")
+                for entity in sorted(items, key=lambda x: x.name):
+                    sections.append(_format_entity(entity))
+
+    # --- PART V: INFRASTRUCTURE ---
+    if infrastructure:
+        sections.append("\n---\n\n# Part V: Infrastructure Indicators\n")
+        by_type = defaultdict(list)
+        for infra in infrastructure:
+            by_type[infra.indicator_type].append(infra)
+
+        for itype in ['ipv4', 'ipv6', 'domain', 'url', 'hash', 'email']:
+            items = by_type.get(itype, [])
+            if not items:
+                continue
+            sections.append(f"\n## {itype.upper()} Indicators\n")
+            sections.append("| Indicator | Context | Source | Date |\n|---|---|---|---|\n")
+            for infra in sorted(items, key=lambda x: x.indicator):
+                ctx = infra.context[:80].replace('|', '/').replace('\n', ' ')
+                sections.append(f"| `{infra.indicator}` | {ctx} | {infra.collection} | {infra.doc_date or 'N/A'} |\n")
+
+    # --- APPENDIX: CROSS-REFERENCES ---
+    sections.append("\n---\n\n# Appendix A: Cross-Reference Index\n")
+    sections.append(_cross_reference_index(tools, ttps, tradecraft, entities))
+
+    # --- APPENDIX: SOURCES ---
+    sections.append("\n---\n\n# Appendix B: Sources\n")
+    sections.append(_sources_appendix(db))
+
+    # Write output
+    content = '\n'.join(sections)
+    out = Path(output_path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(content)
+
+    logger.info("Field manual generated: %s (%d chars)", output_path, len(content))
+    return str(out)
+
+
+def _header(title, stats, tool_count, ttp_count, tc_count, ent_count, infra_count):
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    total_docs = stats.get('total', 0)
+    parsed = stats.get('parsing', {}).get('PARSED', 0)
+
+    return f"""# {title}
+
+> Generated by **Mosaic** — Intelligence Extraction Platform
+> Date: {now}
+> Corpus: {total_docs} documents ({parsed} parsed)
+> Extracted: {tool_count} tools, {ttp_count} TTPs, {tc_count} tradecraft, {ent_count} entities, {infra_count} infrastructure indicators
+
+---
+
+**Classification:** This manual is compiled from publicly available, declassified, and leaked sources.
+Source provenance is noted per entry. Entries sourced from leaked materials (Vault 7, Vault 8) retain
+their original classification markings for reference only.
+
+**Purpose:** Operational reference for cyber and physical tradecraft, techniques, tactics, and
+procedures. Covers the full spectrum from HUMINT field operations to cyber implant deployment.
+
+"""
+
+
+def _toc(tools, ttps, tradecraft, entities, infrastructure):
+    lines = ["## Table of Contents\n"]
+    lines.append("- **Part I: Tradecraft** — Operational methods across all domains")
+    lines.append("- **Part II: TTPs** — Techniques mapped to MITRE ATT&CK where applicable")
+    lines.append("- **Part III: Tools** — Named tools, implants, exploits, and frameworks")
+    lines.append("- **Part IV: Entities** — Organizations, programs, and codenames")
+    if infrastructure:
+        lines.append("- **Part V: Infrastructure** — Network indicators with provenance")
+    lines.append("- **Appendix A: Cross-Reference Index**")
+    lines.append("- **Appendix B: Sources**")
+    return '\n'.join(lines) + '\n'
+
+
+def _format_tradecraft(tc):
+    lines = [f"\n### {tc.method}\n"]
+    lines.append(f"**Domain:** {tc.domain.replace('_', ' ').title()}\n")
+    lines.append(f"{tc.description}\n")
+
+    if tc.operational_notes:
+        lines.append(f"\n**Operational Notes:**\n{tc.operational_notes}\n")
+
+    if tc.countermeasures:
+        lines.append(f"\n**Countermeasures:**\n{tc.countermeasures}\n")
+
+    if tc.source_context:
+        # Truncate for readability
+        ctx = tc.source_context[:300]
+        lines.append(f"\n> *Source: {ctx}*\n")
+
+    return '\n'.join(lines)
+
+
+def _format_ttp(ttp):
+    lines = [f"\n### {ttp.technique}\n"]
+
+    tags = []
+    if ttp.mitre_id:
+        tags.append(f"MITRE: **{ttp.mitre_id}**")
+    if ttp.actor:
+        tags.append(f"Actor: **{ttp.actor}**")
+    tags.append(f"Category: **{ttp.category.replace('_', ' ').title()}**")
+    lines.append(' | '.join(tags) + '\n')
+
+    lines.append(f"\n{ttp.description}\n")
+
+    if ttp.source_context:
+        ctx = ttp.source_context[:300]
+        lines.append(f"\n> *{ctx}*\n")
+
+    return '\n'.join(lines)
+
+
+def _format_tool(tool):
+    lines = [f"\n### {tool.name}\n"]
+
+    if tool.aliases:
+        lines.append(f"**Aliases:** {', '.join(tool.aliases)}\n")
+
+    lines.append(f"**Capability:** {tool.capability.replace('_', ' ').title()}")
+    lines.append(f"**Platforms:** {', '.join(tool.target_platforms) if tool.target_platforms else 'Unknown'}")
+
+    if tool.target_software:
+        lines.append(f"**Targets:** {', '.join(tool.target_software)}")
+
+    if tool.mitre_techniques:
+        lines.append(f"**MITRE:** {', '.join(tool.mitre_techniques)}")
+
+    if tool.cves:
+        lines.append(f"**CVEs:** {', '.join(tool.cves)}")
+
+    lines.append(f"\n{tool.description}\n")
+
+    if tool.source_context:
+        ctx = tool.source_context[:300]
+        lines.append(f"\n> *{ctx}*\n")
+
+    return '\n'.join(lines)
+
+
+def _format_entity(entity):
+    lines = [f"\n### {entity.name}\n"]
+    lines.append(f"**Type:** {entity.type.title()}\n")
+    lines.append(f"{entity.description}\n")
+
+    if entity.relationships:
+        lines.append("\n**Relationships:**")
+        for rel in entity.relationships:
+            lines.append(f"- {rel.get('relationship_type', 'related to').replace('_', ' ')}: **{rel.get('entity', 'Unknown')}**")
+        lines.append("")
+
+    if entity.source_context:
+        ctx = entity.source_context[:200]
+        lines.append(f"\n> *{ctx}*\n")
+
+    return '\n'.join(lines)
+
+
+def _cross_reference_index(tools, ttps, tradecraft, entities):
+    """Build alphabetical index of all terms with section references."""
+    lines = ["\nAlphabetical index of techniques, tools, and tradecraft:\n"]
+
+    all_items = []
+    for t in tools:
+        all_items.append((t.name, "Tool", t.capability.replace('_', ' ').title()))
+        for alias in t.aliases:
+            all_items.append((alias, "Tool (alias)", f"See {t.name}"))
+    for t in ttps:
+        all_items.append((t.technique, "TTP", t.category.replace('_', ' ').title()))
+    for t in tradecraft:
+        all_items.append((t.method, "Tradecraft", t.domain.replace('_', ' ').title()))
+    for e in entities:
+        all_items.append((e.name, "Entity", e.type.title()))
+
+    all_items.sort(key=lambda x: x[0].lower())
+
+    lines.append("| Term | Type | Category/Domain |")
+    lines.append("|---|---|---|")
+    for name, itype, category in all_items:
+        lines.append(f"| {name} | {itype} | {category} |")
+
+    return '\n'.join(lines) + '\n'
+
+
+def _sources_appendix(db):
+    """List all sources with document counts."""
+    lines = ["\nDocument sources used to compile this manual:\n"]
+
+    cursor = db.conn.execute("""
+        SELECT source, COUNT(*) as cnt,
+               SUM(CASE WHEN parse_status='PARSED' THEN 1 ELSE 0 END) as parsed
+        FROM documents
+        GROUP BY source
+        ORDER BY source
+    """)
+
+    lines.append("| Source | Documents | Parsed | Description |")
+    lines.append("|---|---|---|---|")
+
+    source_descriptions = {
+        'vault7': 'CIA Hacking Tools (WikiLeaks Vault 7) — LEAKED',
+        'vault8': 'CIA Source Code (WikiLeaks Vault 8) — LEAKED',
+        'cablegate': 'US Diplomatic Cables (WikiLeaks Cablegate) — LEAKED',
+        'spyfiles': 'Surveillance Industry Documents (WikiLeaks Spy Files) — LEAKED',
+        'emails': 'Email Dumps (WikiLeaks) — LEAKED',
+        'irp_fas': 'FAS Intelligence Resource Program — DECLASSIFIED/PUBLIC',
+        'army_pubs': 'US Army Field Manuals & Regulations — DECLASSIFIED/PUBLIC',
+        'grugq': 'The Grugq OPSEC & Tradecraft Writings — PUBLIC',
+        'cia_rdp': 'CIA Reading Room — DECLASSIFIED',
+        'cryptome': 'Cryptome Document Archive — MIXED',
+    }
+
+    for row in cursor.fetchall():
+        desc = source_descriptions.get(row['source'], 'Custom source')
+        lines.append(f"| {row['source']} | {row['cnt']} | {row['parsed']} | {desc} |")
+
+    lines.append("\n\n**Provenance Key:**")
+    lines.append("- **LEAKED** — Unauthorized disclosure. Original classification markings preserved for reference.")
+    lines.append("- **DECLASSIFIED** — Officially released through FOIA or mandatory declassification review.")
+    lines.append("- **PUBLIC** — Published by author, open source intelligence analysis.")
+
+    return '\n'.join(lines) + '\n'

collectors/base.py

@@ -4,7 +4,7 @@ import tempfile
 import hashlib
 import logging
 from abc import ABC, abstractmethod
-from datetime import datetime
+import datetime
 from pathlib import Path
 from typing import Optional
 
@@ -119,7 +119,7 @@ class BaseCollector(ABC):
                     title=self._extract_title(url),
                     raw_path=final_path,
                     content_hash=content_hash,
-                    fetch_date=datetime.utcnow().isoformat(),
+                    fetch_date=datetime.datetime.now(datetime.timezone.utc).isoformat(),
                     etag=etag,
                     last_modified=last_modified,
                     char_count=len(content.decode('utf-8', errors='replace')),

collectors/custom.py

@@ -91,6 +91,16 @@ class CustomSource(BaseCollector):
             if self._is_excluded(url):
                 continue
 
+            # Scope check: only crawl URLs under the base_url path prefix
+            # This prevents BFS from wandering the entire domain
+            base_path = urlparse(self.base_url).path.rstrip('/')
+            url_path = urlparse(url).path.rstrip('/')
+            if base_path and not url_path.startswith(base_path) and url != self.base_url:
+                # Allow the base URL itself, but skip other paths outside our scope
+                # unless they match a URL pattern (e.g., linked documents)
+                if not self._matches_url_pattern(url):
+                    continue
+
             logger.debug("Crawling: %s (depth %d)", url, depth)
             self.rate_limiter.wait()
 
@@ -100,8 +110,8 @@ class CustomSource(BaseCollector):
                     response.raise_for_status()
                     content_type = response.headers.get('content-type', '').split(';')[0].strip()
 
-                    # If it's a document (not just a page to crawl), add it
+                    # Only collect URLs that match our patterns or are the right content type
-                    if self._matches_url_pattern(url) or self._is_document_type(content_type):
+                    if self._matches_url_pattern(url):
                         item = {
                             'url': url,
                             'title': self._extract_title_from_url(url),

collectors/profiles/army_pubs.yaml

@@ -0,0 +1,24 @@
+name: army_pubs
+description: "US Army field manuals - HUMINT, CI, interrogation, surveillance (public)"
+base_url: "https://irp.fas.org/doddir/army/"
+domain_allowlist:
+  - irp.fas.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/doddir/army/*"
+    - "*.pdf"
+    - "*.htm"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.gif"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 50
+rate_limit: 1.0
+mirrors: []

collectors/profiles/cia_rdp.yaml

@@ -0,0 +1,25 @@
+name: cia_rdp
+description: "CIA Reading Room - Declassified documents (tradecraft, HUMINT, operations)"
+base_url: "https://www.cia.gov/readingroom/collection/what-was-the-cia-doing"
+domain_allowlist:
+  - www.cia.gov
+  - cia.gov
+crawl_rules:
+  max_depth: 3
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/readingroom/*"
+    - "*.pdf"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.png"
+    - "*.jpg"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 50
+rate_limit: 0.5
+mirrors: []

collectors/profiles/cryptome.yaml

@@ -0,0 +1,27 @@
+name: cryptome
+description: "Cryptome - leaked/declassified intelligence documents, tradecraft, surveillance"
+base_url: "https://cryptome.org/"
+domain_allowlist:
+  - cryptome.org
+crawl_rules:
+  max_depth: 2
+  max_documents: 500
+  follow_links: true
+  url_patterns:
+    - "/*.htm"
+    - "/*.pdf"
+    - "/*.txt"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.gif"
+    - "*.jpg"
+    - "*.png"
+content_types:
+  - text/html
+  - application/pdf
+  - text/plain
+expected_min_count: 100
+rate_limit: 0.5
+mirrors: []

collectors/profiles/grugq.yaml

@@ -0,0 +1,22 @@
+name: grugq
+description: "The Grugq's OPSEC and tradecraft writings"
+base_url: "https://grugq.github.io/"
+domain_allowlist:
+  - grugq.github.io
+crawl_rules:
+  max_depth: 2
+  max_documents: 200
+  follow_links: true
+  url_patterns:
+    - "/*"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.png"
+    - "*.jpg"
+    - "*.ico"
+content_types:
+  - text/html
+expected_min_count: 10
+rate_limit: 1.0
+mirrors: []

collectors/profiles/irp_fas.yaml

@@ -0,0 +1,30 @@
+name: irp_fas
+description: "FAS Intelligence Resource Program - declassified manuals, tradecraft, doctrine"
+base_url: "https://irp.fas.org/cia/"
+domain_allowlist:
+  - irp.fas.org
+crawl_rules:
+  max_depth: 3
+  max_documents: 1000
+  follow_links: true
+  url_patterns:
+    - "/cia/*"
+    - "/nsa/*"
+    - "/dia/*"
+    - "/doddir/*"
+    - "*.pdf"
+    - "*.htm"
+    - "*.html"
+  exclude_patterns:
+    - "*.css"
+    - "*.js"
+    - "*.ico"
+    - "*.gif"
+    - "*.png"
+    - "*.jpg"
+content_types:
+  - text/html
+  - application/pdf
+expected_min_count: 100
+rate_limit: 1.0
+mirrors: []

extractors/base.py

@@ -2,7 +2,7 @@
 import json
 import logging
 from abc import ABC, abstractmethod
-from datetime import datetime
+import datetime
 from typing import Optional
 
 import anthropic
@@ -130,7 +130,7 @@ class BaseExtractor(ABC):
 
             # Log token usage
             self.db.log_token_usage(TokenUsage(
-                timestamp=datetime.utcnow().isoformat(),
+                timestamp=datetime.datetime.now(datetime.timezone.utc).isoformat(),
                 source=source,
                 doc_id=doc_id,
                 extractor=self.EXTRACTOR_NAME,

mosaic.py

@@ -158,17 +158,7 @@ def _load_profiles_fallback() -> list[dict]:
 # Click CLI
 # ---------------------------------------------------------------------------
 
-class MosaicCLI(click.Group):
+@click.group(invoke_without_command=True)
-    """Custom group that shows the interactive menu when invoked bare."""
-
-    def invoke(self, ctx):
-        if not ctx.protected_args and not ctx.invoked_subcommand:
-            interactive_menu(ctx)
-        else:
-            super().invoke(ctx)
-
-
-@click.group(cls=MosaicCLI, invoke_without_command=True)
 @click.option("--source", type=str, default=None, help="Source profile name.")
 @click.option("--url", type=str, default=None, help="Ad-hoc source URL.")
 @click.option("--profile", "profile_path", type=click.Path(exists=True), default=None,
@@ -205,6 +195,9 @@ def cli(ctx, source, url, profile_path, dry_run, retry_failed, since, date_range
     ctx.obj["verbose"] = verbose
     ctx.obj["output_dir"] = get_output_dir(cfg)
 
+    if ctx.invoked_subcommand is None:
+        interactive_menu(ctx)
+
 
 # ---------------------------------------------------------------------------
 # collect
@@ -231,10 +224,28 @@ def collect(ctx, source, url, name, profile_path, limit, since, date_range, upda
         console.print("[yellow]No source specified.[/yellow] Use --source, --url, or --profile.")
         raise SystemExit(1)
 
-    label = source or url or profile_path
+    from collectors.profile_loader import load_profile, create_adhoc_profile
+    from collectors.custom import CustomSource
+    from utils.cache import DownloadCache
+    from utils.http_limiter import HTTPRateLimiter
+
+    # Load profile
+    try:
+        if profile_path:
+            profile = load_profile(profile_path)
+        elif url:
+            pname = name or url.split('/')[2].replace('.', '_')
+            profile = create_adhoc_profile(url, pname)
+        else:
+            profile = load_profile(source)
+    except (FileNotFoundError, ValueError) as e:
+        console.print(f"[red]Profile error: {e}[/red]")
+        raise SystemExit(1)
+
+    label = profile['name']
     console.print(Panel(
         f"[bold cyan]Collect[/bold cyan]\n\n"
-        f"  Source : {label}\n"
+        f"  Source : {label} — {profile.get('description', '')}\n"
         f"  Limit  : {limit or 'all'}\n"
         f"  Since  : {since or 'n/a'}\n"
         f"  Range  : {' to '.join(date_range) if date_range else 'n/a'}\n"
@@ -242,9 +253,40 @@ def collect(ctx, source, url, name, profile_path, limit, since, date_range, upda
         title="[bold]mosaic collect[/bold]",
         border_style="cyan",
     ))
-    console.print(
+
-        "\n[dim]Collection not yet implemented — see collectors/[/dim]"
+    db = open_db(ctx.obj["db_path"])
-    )
+    try:
+        cache = DownloadCache(os.path.join(ctx.obj["output_dir"], "raw"))
+        cfg = ctx.obj["cfg"]
+        http_cfg = {
+            'timeout': cfg.get('http', 'timeout', fallback='30'),
+            'max_retries': cfg.get('http', 'max_retries', fallback='3'),
+            'user_agent': cfg.get('http', 'user_agent', fallback='Mozilla/5.0 (compatible; research-crawler/1.0)'),
+        }
+        collector = CustomSource(profile, db, cache, config=http_cfg)
+
+        since_date = since
+        until_date = None
+        if date_range:
+            since_date, until_date = date_range
+
+        stats = collector.collect(
+            since=since_date, until=until_date,
+            limit=limit, update=update,
+        )
+
+        table = Table(title="Collection Results", box=box.ROUNDED)
+        table.add_column("Metric", style="bold")
+        table.add_column("Count", justify="right")
+        table.add_row("Discovered", str(stats.get('discovered', 0)))
+        table.add_row("Downloaded", str(stats.get('downloaded', 0)))
+        table.add_row("Already cached", str(stats.get('cached', 0)))
+        table.add_row("Failed", str(stats.get('failed', 0)))
+        if update:
+            table.add_row("Updated", str(stats.get('updated', 0)))
+        console.print(table)
+    finally:
+        db.close()
 
 
 # ---------------------------------------------------------------------------
@@ -260,6 +302,9 @@ def parse(ctx, source, retry_failed):
     source = source or ctx.obj.get("source")
     retry_failed = retry_failed or ctx.obj.get("retry_failed", False)
 
+    from parsers.base import ParserManager
+    from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+
     console.print(Panel(
         f"[bold green]Parse[/bold green]\n\n"
         f"  Source      : {source or 'all'}\n"
@@ -267,9 +312,80 @@ def parse(ctx, source, retry_failed):
         title="[bold]mosaic parse[/bold]",
         border_style="green",
     ))
-    console.print(
+
-        "\n[dim]Parsing not yet implemented — see parsers/[/dim]"
+    db = open_db(ctx.obj["db_path"])
-    )
+    try:
+        cfg = ctx.obj["cfg"]
+        parser = ParserManager(
+            memory_limit_mb=int(cfg.get('parser', 'subprocess_memory_limit_mb', fallback='512')),
+            timeout_seconds=int(cfg.get('parser', 'subprocess_timeout_seconds', fallback='60')),
+            min_chars_per_page=int(cfg.get('parser', 'min_chars_per_page', fallback='50')),
+            max_pdf_pages=int(cfg.get('parser', 'max_pdf_pages', fallback='200')),
+        )
+
+        # Get documents to parse
+        if source:
+            docs = db.get_documents_by_source(source, status='CACHED')
+        else:
+            docs = db.get_documents_by_source('', status='CACHED')
+            # Get all cached docs across sources
+            cursor = db.conn.execute(
+                "SELECT * FROM documents WHERE status = 'CACHED' AND parse_status IN ('PENDING', ?)",
+                ('PARSE_FAILED' if retry_failed else '__NONE__',)
+            )
+            from storage.models import Document
+            docs = [Document.from_row(row) for row in cursor.fetchall()]
+
+        if not docs:
+            console.print("[yellow]No documents to parse.[/yellow]")
+            return
+
+        parsed = 0
+        failed = 0
+        needs_ocr = 0
+
+        with Progress(
+            SpinnerColumn(), TextColumn("{task.description}"),
+            BarColumn(), TaskProgressColumn(),
+        ) as progress:
+            task = progress.add_task(f"Parsing {len(docs)} docs", total=len(docs))
+            for doc in docs:
+                if doc.parse_status in ('PARSED', 'NEEDS_OCR') and not retry_failed:
+                    progress.advance(task)
+                    continue
+
+                db.update_document_status(doc.id, parse_status='IN_PROGRESS')
+                result = parser.parse(doc.raw_path, doc.doc_type, doc.content_hash)
+
+                if result.success:
+                    doc.text = result.text
+                    doc.title = result.title or doc.title
+                    doc.char_count = result.char_count
+                    if result.metadata:
+                        doc.metadata.update(result.metadata)
+                    ps = 'NEEDS_OCR' if result.needs_ocr else 'PARSED'
+                    doc.parse_status = ps
+                    db.insert_document(doc)
+                    db.update_document_status(doc.id, parse_status=ps)
+                    if result.needs_ocr:
+                        needs_ocr += 1
+                    else:
+                        parsed += 1
+                else:
+                    db.update_document_status(doc.id, parse_status='PARSE_FAILED')
+                    failed += 1
+
+                progress.advance(task)
+
+        table = Table(title="Parse Results", box=box.ROUNDED)
+        table.add_column("Metric", style="bold")
+        table.add_column("Count", justify="right")
+        table.add_row("Parsed", str(parsed))
+        table.add_row("Needs OCR", str(needs_ocr))
+        table.add_row("Failed", str(failed))
+        console.print(table)
+    finally:
+        db.close()
 
 
 # ---------------------------------------------------------------------------
@@ -710,6 +826,35 @@ def _run_verify(db, output_dir: str, source: Optional[str] = None):
         console.print("\n[green]All cached files passed integrity checks.[/green]")
 
 
+# ---------------------------------------------------------------------------
+# manual
+# ---------------------------------------------------------------------------
+
+@cli.command()
+@click.option("--output", "-o", type=click.Path(), default="output/FIELD_MANUAL.md",
+              help="Output file path.")
+@click.option("--title", type=str, default="Operator's Field Manual",
+              help="Manual title.")
+@click.pass_context
+def manual(ctx, output, title):
+    """Generate Markdown field manual from extracted intelligence."""
+    from analysis.field_manual import generate_field_manual
+
+    db = open_db(ctx.obj["db_path"])
+    try:
+        path = generate_field_manual(db, output_path=output, title=title)
+        console.print(f"\n[bold green]Field manual generated:[/bold green] {path}")
+
+        # Show stats
+        import os
+        size = os.path.getsize(path)
+        with open(path) as f:
+            lines = sum(1 for _ in f)
+        console.print(f"  Size: {size:,} bytes ({lines:,} lines)")
+    finally:
+        db.close()
+
+
 # ---------------------------------------------------------------------------
 # sources
 # ---------------------------------------------------------------------------

output/manuals/01_HUMINT_Operations.md

@@ -0,0 +1,439 @@
+# HUMINT Operations Manual
+
+> Synthesized from: Allen Dulles "Some Elements of Intelligence Work," FM 2-22.3, JSOU Report 12-3,
+> CIA tradecraft manuals (declassified), Grugq OPSEC analyses
+>
+> Classification: OPEN SOURCE — compiled from publicly available and declassified materials
+
+---
+
+## Table of Contents
+
+1. Foundations
+2. The Agent Recruitment Cycle
+3. Elicitation
+4. Agent Handling & Communication
+5. Cover & Legend Discipline
+6. Clandestine Organization Structure
+7. Counter-Intelligence
+8. Case Studies & Lessons Learned
+9. Operational Checklists
+
+---
+
+# Chapter 1: Foundations
+
+## 1.1 The Nature of HUMINT
+
+Human Intelligence is the oldest form of intelligence collection. It is also the most dangerous, the most unreliable, and the most valuable. Every other collection discipline — SIGINT, IMINT, MASINT, OSINT — tells you what is happening. HUMINT tells you *why*, and sometimes *what will happen next*.
+
+HUMINT operations rest on a simple premise: people with access to secrets can be persuaded to share them. The persuasion may take many forms — ideology, money, coercion, ego, revenge — but the fundamental transaction is always human. This makes HUMINT operations simultaneously powerful and fragile. A single relationship built over years can produce intelligence that no satellite or intercept can match. A single mistake — a careless phone call, an unnoticed surveillance team, a compromised colleague — can destroy an entire network and cost lives.
+
+## 1.2 The Dulles Principles
+
+Allen Dulles, director of the CIA from 1953 to 1961, wrote a set of operational principles that remain the foundation of Western intelligence tradecraft. These principles, titled "Some Elements of Intelligence Work," distill decades of operational experience into rules that are deceptively simple and absolutely non-negotiable.
+
+**The Cardinal Rules:**
+
+1. **Security above all.** "The greatest of them all is security. All else must be subordinated to that." This is not a suggestion. It is the single principle from which all others flow. Every decision in an operation must be evaluated first through the lens of security.
+
+2. **Security is in the small things.** "It consists in carrying out daily tasks with painstaking remembrance of the tiny things that security demands. The little things are in many ways more important than the big ones. It is they which oftenest give the game away." The operative who leaves a classified document on a desk for thirty seconds, who mentions a colleague's name to a taxi driver, who forgets to check a mirror on the way to a meeting — this operative will eventually be caught. Security is not an occasional effort. It is a permanent state of mind.
+
+3. **Practice makes permanent.** "The man or woman who does not indulge in the daily security routine, boring and useless though it may sometimes appear, will be found lacking in the proper instinctive reaction when dealing with the bigger stuff." When the pressure comes — and it will — you will fall back on your training. If your training is sloppy, your reaction will be sloppy. Practice security routines until they are automatic. Then practice more.
+
+4. **Never admit.** "Even though you feel the curious outsider has probably a good idea that you are not what you purport to be, never admit it. Keep on playing the other part. It's amazing how often people will be led to think they were mistaken." Cover is maintained not by perfection but by consistency. People doubt their own suspicions. Give them reason to keep doubting.
+
+5. **Vanity kills.** "The next greatest vice is that of vanity. Its offshoots are multiple and malignant. Besides, the man with a swelled head never learns." The operative who needs to feel clever, who drops hints about their secret life, who takes unnecessary risks to prove their capability — this operative is a liability. The best operatives are invisible. They have no need for recognition.
+
+6. **No hours.** "In this job, there are no hours. That is to say, one never leaves it down. It is lived. One never drops one's guard." Every social occasion is an opportunity to lay a false trail, to pick up information, to make a useful acquaintance. The operative who compartmentalizes their life into "on duty" and "off duty" has already made a fundamental error. You are always on.
+
+7. **The telephone is the enemy.** "The greatest material curse to the profession, despite all its advantages, is undoubtedly the telephone. It is a constant source of temptation to slackness." Always assume every conversation is listened to. Never discuss operations by phone. If you must use a phone, think out what you will say before you dial. Better yet: "Make a day's journey, rather than take a risk, either by phone or post."
+
+8. **Alcohol and sex are operational vulnerabilities.** "Booze is naturally dangerous. So also is an undisciplined attraction for the other sex. The first loosens the tongue. The second does likewise. It also distorts vision and promotes indolence." This is not moralizing. It is operational reality. Both are primary vectors for compromise and recruitment by hostile services.
+
+9. **Write it down, but carefully.** "When you have conducted an interview or made arrangements for a meeting, write it all down and put it safely away for reference. Your memory can play tricks." But: "Learn to write lightly; the 'blank' page underneath has often been read. Be wary of your piece of blotting paper. If you have to destroy a document, do so thoroughly. Carry as little written matter as possible, and for the shortest possible time. Never carry names or addresses en clair."
+
+10. **Carelessness is irreversible.** "The greatest vice in the game is that of carelessness. Mistakes made generally cannot be rectified." In most professions, you can recover from a mistake. In intelligence, a single error can burn a network that took a decade to build.
+
+---
+
+# Chapter 2: The Agent Recruitment Cycle
+
+## 2.1 Overview: SADRAT
+
+The recruitment of a human intelligence source follows a cycle known by various mnemonics. The most common is SADRAT:
+
+- **S**pot — Identify potential sources
+- **A**ssess — Evaluate suitability and vulnerability
+- **D**evelop — Build the relationship
+- **R**ecruit — Make the pitch
+- **A**gent handling — Run the source
+- **T**erminate — End the relationship
+
+Each phase has distinct tradecraft requirements and risks. Rushing through phases or skipping them entirely is the most common cause of recruitment failure.
+
+## 2.2 Spotting
+
+Spotting is the identification of individuals who have access to desired intelligence and may be susceptible to recruitment. Good spotting is the foundation of everything that follows. Recruit the wrong person and you waste months or years. Recruit the right person and you win the intelligence war.
+
+**Access is primary.** The most cooperative source in the world is useless if they cannot access the intelligence you need. Before evaluating personality, motivation, or vulnerability, answer one question: does this person have access?
+
+**Types of access:**
+- Direct access — the person handles, sees, or creates the target information
+- Indirect access — the person knows someone who has direct access
+- Environmental access — the person is in a position to observe the target environment
+
+**Spotting methods:**
+1. **Official contact** — Diplomatic receptions, conferences, trade shows, academic exchanges. These provide natural cover for initial contact.
+2. **Third-party referral** — An existing source identifies a potential new source. This is often the most productive method but carries risk if the referrer is compromised.
+3. **Cold approach analysis** — Study of organizational charts, publications, social media, and public records to identify officials with access. This produces candidates who may never have been contacted by an intelligence service.
+4. **Walk-ins** — Volunteers who approach with an offer. These are simultaneously the most valuable and the most dangerous potential sources. Every walk-in must be assessed for the possibility that they are a dangle — a controlled source sent by hostile counterintelligence.
+
+## 2.3 Assessment
+
+Assessment determines whether a spotted individual is suitable for recruitment and how to approach them. The assessment phase can last weeks, months, or years depending on the target's significance and the operational environment.
+
+**The MICE Framework**
+
+The classic framework for understanding recruitment motivation:
+
+**Money** — Financial pressure, greed, lifestyle expectations beyond income. The most common and most reliable motivator. People who need money are predictable. People who want money are manageable. Look for: living beyond means, gambling debts, expensive divorce, addiction, sudden financial obligations.
+
+**Ideology** — Belief that sharing information serves a higher purpose. The volunteer spy who believes their government is corrupt, their cause is just, or their information will prevent harm. Ideological recruits are often the most productive but also the most unpredictable. They may decide at any time that the higher purpose no longer justifies the risk. Look for: political dissatisfaction, moral objections to employer's activities, ideological alignment with your cause.
+
+**Compromise/Coercion** — Leverage based on information the target wants kept secret. Extramarital affairs, financial crimes, hidden political affiliations, sexual orientation in hostile environments. Coercion produces immediate compliance but breeds resentment and unreliability. Use it as a last resort and always provide a face-saving framework. "We're not forcing you — we're giving you the opportunity to do the right thing."
+
+**Ego** — The need to feel important, respected, or appreciated. The mid-level bureaucrat who has been passed over for promotion, the technical expert whose contributions go unrecognized, the official who believes they should be making policy rather than implementing it. Ego-driven sources are managed by making them feel valued. "You're the only person who really understands this issue."
+
+**Modern additions to MICE:**
+- **Revenge** — Personal grievance against employer, colleague, or institution
+- **Adventure** — Thrill-seeking, desire for excitement in a boring life
+- **Conscience** — Similar to ideology but more personal; the whistleblower who cannot live with what they know
+
+**Assessment process:**
+1. Identify the primary motivator (there may be more than one)
+2. Evaluate access level and quality
+3. Assess reliability indicators (stability, discretion, consistency)
+4. Identify vulnerabilities that hostile CI could exploit
+5. Evaluate the risk/reward ratio of recruitment
+6. Develop an approach strategy tailored to the individual
+
+## 2.4 Development
+
+Development is the gradual building of a relationship that will support a recruitment pitch. This is where patience is tested and careers are made. The development phase transforms a target from a stranger into someone who trusts you enough to commit espionage.
+
+**Principles of development:**
+
+1. **Natural contact.** Every meeting must have a plausible reason unrelated to intelligence. Business, social, academic, cultural — the reason must be genuine enough to withstand scrutiny.
+
+2. **Gradual escalation.** Start with innocuous topics. Gradually introduce subjects closer to the target's area of knowledge. Never jump from small talk to classified information. The progression should feel natural to the target.
+
+3. **Reciprocity.** Share information to get information. People who feel they are in a one-sided conversation become suspicious. Provide non-sensitive information that makes the target feel the exchange is balanced.
+
+4. **Build dependency.** Create a relationship where the target values your friendship, your contacts, your resources, or your understanding. By the time you make the pitch, the target should feel that saying no would damage something they value.
+
+5. **Test incrementally.** Before the formal pitch, test the target's willingness to share sensitive information through increasingly specific questions. If they share willingly, they may already be psychologically recruited before the formal ask.
+
+## 2.5 Recruitment
+
+The recruitment pitch is the moment of maximum risk and maximum opportunity. Everything that follows depends on what happens in this conversation.
+
+**Before the pitch:**
+- Ensure you have completed a thorough SDR (Surveillance Detection Route) — see Manual 02
+- Choose a location that is private, secure, and offers plausible cover for the meeting
+- Have a clear understanding of what you are asking and what you are offering
+- Prepare for all possible responses: acceptance, rejection, anger, panic, negotiation
+- Have a fallback position if the initial pitch is rejected
+
+**The pitch itself:**
+
+1. **Frame the ask.** Do not say "I want you to spy for us." Frame it in terms the target will accept: "I'd like to formalize our information exchange," "Your insights are valuable and I want to make sure they reach the right people," "There's an opportunity for you to make a real difference."
+
+2. **Appeal to the identified motivator.** If money: be specific about compensation. If ideology: emphasize the impact of their contribution. If ego: stress how uniquely qualified they are. If compromise: present the choice as the target having agency, not being coerced.
+
+3. **Establish the rules.** Communication methods, meeting schedules, security protocols, reporting requirements. The target needs structure. Ambiguity breeds anxiety, and anxious sources make mistakes.
+
+4. **Address fear.** Every potential source is afraid. Afraid of being caught, of prison, of shame, of violence. Acknowledge the fear. Explain the security measures that protect them. Do not minimize the risk — they know the risk. Instead, demonstrate competence in managing it.
+
+5. **Get a commitment.** The pitch must end with a clear answer and a clear next step. "Can I count on you?" followed by "Our next meeting will be..."
+
+## 2.6 Agent Handling
+
+Once recruited, a source must be managed — tasked, debriefed, paid, motivated, and protected. The handler-source relationship is the backbone of HUMINT operations.
+
+**Communication:**
+- Establish primary and backup communication methods (see Manual 03: Covert Communications)
+- Set regular meeting schedules with security protocols for cancellation/emergency
+- Use dead drops for routine material, personal meetings for tasking and debriefing
+- Every meeting preceded by SDR (see Manual 02)
+
+**Tasking:**
+- Be specific about what you need. Vague requirements produce useless intelligence.
+- Prioritize. Do not overwhelm the source with requests.
+- Explain why the information matters when possible — motivated sources produce better intelligence.
+- Never task a source to do something that increases their exposure beyond acceptable risk.
+
+**Security:**
+- Compartment the source's identity. Minimize the number of people who know who they are.
+- Vary meeting locations, times, and methods.
+- Monitor for changes in the source's behavior, access, or environment that might indicate compromise.
+- Have an exfiltration plan. If the source is compromised, you must be able to get them out.
+
+---
+
+# Chapter 3: Elicitation
+
+## 3.1 What Elicitation Is
+
+Elicitation is the extraction of information through conversation without the target realizing that intelligence collection is taking place. It is the art of getting people to tell you things they shouldn't, while believing they are having a normal conversation.
+
+Unlike interrogation, which is overt, elicitation is covert. The target never knows they were debriefed. This makes elicitation both repeatable and deniable.
+
+## 3.2 Core Techniques
+
+**Flattery / Appeal to expertise.** "You're clearly the person who understands this best. How does the new system actually work?" People love to demonstrate their knowledge. Position yourself as impressed and slightly ignorant. The target will fill the gap.
+
+**Provocative statement.** State something slightly wrong about the target's area of expertise. "I heard the new encryption protocol only covers external communications." The target's instinct to correct you will produce real information. "Actually, it covers everything including internal traffic between..."
+
+**Assumed knowledge.** Speak as if you already know most of the answer, but leave a gap. "So the deployment in the northern district uses the standard configuration, but I've heard the southern district has a different setup..." The target confirms, denies, or elaborates — all useful.
+
+**Quid pro quo.** Share something first. It doesn't have to be classified or even sensitive — just interesting enough to create a sense of reciprocal obligation. "We've been seeing unusual network traffic from [country X]. Have you noticed anything similar?"
+
+**Bracketing.** State a range and let the target narrow it. "The project budget is somewhere between five and fifty million, right?" The target will instinctively narrow: "It's closer to twenty." Now you know.
+
+**Deliberate ignorance.** Play completely dumb. Ask simple, open-ended questions that require detailed explanation. "I've never understood how your organization's approval process works. Can you walk me through it?" People explaining processes reveal structure, personnel, and vulnerabilities.
+
+**The drunk friend.** Not about actually being drunk, but about creating an atmosphere of casual, unguarded conversation. Late-night conferences, after-dinner drinks, informal social gatherings. People say more when they feel the conversation is "off the record."
+
+## 3.3 Principles
+
+1. **Never ask a direct question about the target information.** Direct questions trigger security awareness. Indirect approaches bypass it.
+2. **Let the target feel like the expert.** You are the student. They are the teacher. This flatters their ego and makes them want to demonstrate knowledge.
+3. **Multiple sessions.** One conversation rarely produces actionable intelligence. Build the picture over multiple interactions. Each conversation adds fragments.
+4. **Social settings.** Alcohol genuinely does lower inhibitions. Use social occasions strategically, but never lose your own control while trying to reduce theirs.
+5. **Listen more than you talk.** The target should be doing 70-80% of the talking. Your role is to guide, not interrogate.
+
+---
+
+# Chapter 4: Agent Handling & Communication
+
+## 4.1 Communication Methods
+
+The most dangerous part of any intelligence operation is communication between handler and source. This is when both parties are most exposed. Every communication method involves a trade-off between security and convenience. Always err on the side of security.
+
+**Hierarchy of communication security (most to least secure):**
+1. Dead drop with pre-arranged signal (no contact, no timing correlation)
+2. Brush pass in public (sub-second contact, no meeting)
+3. Personal meeting at secure location after SDR (necessary for debriefing)
+4. Courier via trusted third party
+5. SRAC (Short Range Agent Communications) device
+6. Encrypted digital communication (creates metadata)
+7. Telephone (never for operational content)
+
+See Manual 03: Covert Communications for detailed procedures.
+
+## 4.2 Meeting Protocols
+
+**Pre-meeting:**
+1. Conduct full SDR (2-4 hours minimum in hostile environment)
+2. Confirm "black" status (no surveillance detected)
+3. Arrive at meeting location via planned route
+4. If meeting is canceled (danger signal observed), execute emergency protocol
+
+**During meeting:**
+1. Keep meetings short. The longer you are together, the greater the risk.
+2. Cover story for the meeting must be plausible if observed.
+3. Debrief first (collect intelligence), then task (assign new requirements).
+4. Address source morale and security concerns.
+5. Handle payments if applicable.
+6. Set next meeting date, time, location, and backup.
+7. Establish danger signals for next meeting.
+
+**Post-meeting:**
+1. Depart separately, via different routes.
+2. Conduct counter-surveillance on departure.
+3. Write up meeting notes at the earliest secure opportunity.
+4. File intelligence reports through proper channels.
+5. Assess any security concerns from the meeting.
+
+---
+
+# Chapter 5: Cover & Legend Discipline
+
+See Manual 05: Cover and Identity for comprehensive treatment. Key principles:
+
+## 5.1 Cover Levels
+
+- **Light cover:** Alias name only. Suitable for brief, low-risk contacts.
+- **Medium cover:** Alias with supporting documentation (ID, business cards, backstopped phone number). Suitable for extended contact in permissive environments.
+- **Deep cover:** Full legend with years of backstory, employment history, social media presence, and established pattern of life. Required for hostile environments and long-term penetration operations.
+
+## 5.2 The Dulles Rule on Cover
+
+"Do not overwork your cover to the detriment of your jobs; we must never get so engrossed in the latter as to forget the former."
+
+Cover is not an afterthought. It is the foundation that makes everything else possible. An operative without solid cover is a tourist, not a spy.
+
+---
+
+# Chapter 6: Clandestine Organization Structure
+
+## 6.1 Cell Structure
+
+The fundamental unit of clandestine organization is the cell: a small group of individuals who work together and are isolated from other cells by compartmentation. Cell structure provides resilience against penetration. When one cell is compromised, the damage stops at the cell boundary.
+
+**Structural compartmentation:** Each cell knows only its adjacent cells. A member of Cell A knows Cell A's members and has contact with one member of Cell B. That is all. Cell A does not know that Cell C exists.
+
+**Functional compartmentation:** Tradecraft practices that minimize the operational signature of the organization. Even within a cell, members know only what they need to know for their specific function.
+
+**Key insight from JSOU Report 12-3:** In the Iraqi insurgency, RAND analyst Bruce Hoffman initially concluded the insurgency was "uncoordinated and disconnected" with "no static wiring diagram." He was wrong. The insurgency was a tightly coordinated clandestine cellular network that applied excellent tradecraft to remain hidden. The visible parts of the network — the cells at the periphery that conducted attacks — practiced poor tradecraft and were detected. But the invisible connections between cells, protected by cut-outs and compartmentation, remained intact. When a frontline cell was destroyed, "a new one will take its place within a couple of weeks at the most."
+
+## 6.2 Cut-Outs and Impersonal Communication
+
+Cut-outs are mechanisms that prevent direct contact between members of different cells. They ensure that no single compromise can chain through the network.
+
+**Passive methods (lower signature):**
+- Couriers — most secure method of moving messages and materials. Women and children are often used because they attract less suspicion at checkpoints.
+- Dead drops — one party deposits, another retrieves, no contact (see Manual 03)
+- Coded signals — visual indicators that convey pre-arranged messages
+
+**Active methods (higher signature):**
+- Radio (detectable emissions)
+- Telephone (metadata collection)
+- Internet (traffic analysis)
+
+The general principle: passive methods for high-threat environments, active methods only when the speed of communication justifies the increased risk.
+
+## 6.3 Regeneration
+
+Well-designed clandestine networks can regenerate destroyed cells because the organizational knowledge is distributed, not centralized. The cells that face the enemy are expendable. The deep network survives and replaces losses. This is why targeting only the visible cells of an insurgency or criminal organization is futile — you must penetrate the hidden infrastructure.
+
+---
+
+# Chapter 7: Counter-Intelligence
+
+## 7.1 The Informant Threat
+
+The single greatest threat to any clandestine organization is the informant — a member who reports to the opposition. Technical surveillance can be defeated with tradecraft. Informants bypass all technical countermeasures because they are inside the organization.
+
+As the Grugq writes: **"When in doubt, it's a tout."**
+
+When a group with robust operational security practices is compromised, and the technical indicators don't explain how, the answer is almost always an informant.
+
+## 7.2 The Reservoir Dogs SOP (Anti-Informant Tradecraft)
+
+The Grugq analyzed the OPSEC procedures depicted in the film *Reservoir Dogs*, noting they are based on real SOPs used by Fatah and the Black September Organisation (BSO). These procedures provide effective protection against informants within an operational team:
+
+**Procedure 1: Assigned operational aliases.** Aliases are assigned by the organization for the duration of the operation. They are random, unique to the operation, and not chosen by the operative. This prevents pattern development across operations and limits the information an informant can gather.
+
+**Procedure 2: Just-in-time team assembly.** The operational team is formed immediately before the operation and kept isolated until after completion. This minimizes the time window for an informant to report back to their handlers.
+
+**Procedure 3: Dedicated independent operational support teams.** Surveillance, logistics, and execution are handled by separate teams. Each team knows only their portion of the plan. Compromise of one team does not reveal the complete operation.
+
+**Strengths:** Limits informant intelligence, protects agents until activation, provides operational compartmentation.
+
+**Weaknesses:** Ad-hoc teams are less efficient (compressed forming-storming-norming-performing cycle). The organizational leadership who form teams become high-value targets — they know everyone. The team captain is a single point of failure — only person with the complete plan.
+
+## 7.3 CI Awareness Indicators
+
+Signs that a member may be compromised or may be an informant:
+- Unexplained knowledge of operations they weren't briefed on
+- Changes in financial status inconsistent with known income
+- Unusual interest in operational details beyond their need-to-know
+- Behavioral changes (anxiety, evasion, sudden eagerness to please)
+- Unexplained absences that correlate with opposition activity
+- Advocating for reduced security measures
+
+---
+
+# Chapter 8: Case Studies & Lessons Learned
+
+## 8.1 PIRA — The Paddy Factor
+
+The Provisional IRA in the early 1970s demonstrated how basic counter-intelligence failures can devastate an organization.
+
+**The failures:**
+- Members congregated in pubs and sang IRA songs, publicly identifying themselves
+- Members boasted about operations while drunk
+- Members responded to inquiries about their activities with "a nod and a wink"
+- Members marched in pro-IRA rallies
+- Members associated with each other when not on operations (pre-operational contact)
+
+**The consequence:** British security forces identified known PIRA members through their public behavior. They then monitored these known members. When known members socialized with unknown members at rallies and pubs, the unknown members were identified through surveillance. Link analysis — mapping associations between nodes in a network — expanded the identified membership rapidly.
+
+**The lesson:** Self-incrimination through public behavior is the most basic CI failure. It doesn't matter how strong your operational security is if your members advertise their affiliation in public. Pre-operational contact — socializing with operational colleagues outside of operations — creates links that surveillance can exploit. One known member in a social group exposes all unknown members.
+
+## 8.2 Ross Ulbricht / Dread Pirate Roberts — Persona Contamination
+
+The Silk Road case demonstrates what happens when compartmentation between identities fails.
+
+**The fatal chain:**
+1. Ulbricht created the "altoid" persona to promote Silk Road on forums (Shroomery, BitcoinTalk)
+2. The altoid persona had no backstopping — no dedicated email address — so Ulbricht eventually posted his personal email (rossulbricht@gmail.com) on BitcoinTalk
+3. The "frosty" persona used to administer the Silk Road server was accessed from a location 500 feet from Ulbricht's Gmail login location
+4. Both Ulbricht and DPR shared the same ideology (Austrian School of Economics, mises.org), timezone, and geographic location
+5. Social isolation drove Ulbricht to seek validation on forums, lowering his security discipline
+
+**The lesson:** Compartmentation between personas must be absolute. A single link — one shared email, one shared ideology publicly expressed, one shared location — can chain identities together. Backstop every persona. Never post personal identifiers from operational personas. And critically: isolation degrades judgment. Underground operatives who have no social life outside their operational community lose perspective and make mistakes.
+
+## 8.3 Robert Morris — STFU
+
+Robert Morris created the first major internet worm in 1988. He was identified and prosecuted not through technical forensics alone, but because he couldn't stop talking about it.
+
+**The failure:** Morris briefed his friends — the "Morris Cell" — on all aspects of the worm: how it was developed, how it worked, what vulnerabilities it exploited. He did this at restaurants and social gatherings, including one incident where he jumped on a table to explain the worm's mechanics. His friends were subpoenaed as prosecution witnesses. They had no choice but to testify.
+
+**The lesson:** **STFU.** Unless your lawyer instructs you otherwise, say nothing. The rule of thumb for need-to-know: **if someone is not actively sharing the risk, they have no need to know.** Even those who are sharing the risk should know only the aspects of the operation in which they are directly involved.
+
+## 8.4 CIA Lebanon — Real-World Anonymity Failure
+
+In late 2011, Hezbollah rolled up a CIA spy ring in Lebanon. The failure was not in encryption or digital security — it was in real-world tradecraft.
+
+**What happened:** CIA agents had dedicated mobile phones used specifically for communication with their handlers. The phones were kept at static locations. Meetings occurred at pre-arranged locations (reportedly a Pizza Hut). Hezbollah's counter-intelligence identified the pattern: dedicated devices, static locations, predictable meeting schedules.
+
+**The lesson:** Encryption protects content. It does not protect against traffic analysis, pattern analysis, or behavioral profiling. A dedicated device that only activates for handler communication is itself an anomaly. A static meeting location creates a pattern. Anonymity — hiding the fact that communication is occurring at all — is more important than secrecy of the communication's content.
+
+---
+
+# Chapter 9: Operational Checklists
+
+## 9.1 Pre-Recruitment Assessment Checklist
+
+- [ ] Target has confirmed access to desired intelligence
+- [ ] Primary motivation identified (MICE + additional factors)
+- [ ] Secondary motivations identified
+- [ ] Vulnerability assessment complete
+- [ ] Background investigation shows no CI red flags
+- [ ] Target is not a known or suspected dangle
+- [ ] Approach strategy developed
+- [ ] Cover for initial contact is plausible
+- [ ] Risk/reward assessment documented
+- [ ] Exfiltration plan exists if recruitment fails badly
+
+## 9.2 Agent Meeting Checklist
+
+- [ ] SDR conducted (minimum 2 hours in hostile environment)
+- [ ] Confirmed "black" — no surveillance detected
+- [ ] Meeting location is secure and offers plausible cover
+- [ ] Danger signals established with source
+- [ ] Debriefing objectives prepared
+- [ ] New tasking prepared
+- [ ] Payment ready if applicable
+- [ ] Next meeting date/time/location/backup prepared
+- [ ] Counter-surveillance planned for departure
+- [ ] Emergency protocol reviewed
+
+## 9.3 CI Awareness Checklist
+
+- [ ] All members trained on elicitation recognition
+- [ ] Reporting procedures for unusual contacts established
+- [ ] Financial monitoring in place for access personnel
+- [ ] Need-to-know enforced for all operational information
+- [ ] Pre-operational contact minimized
+- [ ] Public affiliation indicators eliminated
+- [ ] Communication security protocols followed
+- [ ] Regular security briefings conducted
+- [ ] Informant detection indicators monitored
+
+---
+
+*This manual is a living document. Regenerate with `python mosaic.py manual` as new sources are collected and extracted.*

output/manuals/04_Physical_Access.md

@@ -0,0 +1,390 @@
+# Physical Access & Entry Manual
+
+> Synthesized from: CIA tradecraft manuals (declassified), physical security assessment resources,
+> Grugq OPSEC analyses, FM 2-22.3, open-source tradecraft literature
+>
+> Classification: OPEN SOURCE — compiled from publicly available materials
+
+---
+
+## Table of Contents
+
+1. Principles of Physical Access
+2. Target Assessment
+3. Lock Bypass Techniques
+4. Social Engineering for Access
+5. Surreptitious Entry Operations
+6. Postal & Logistics Operations
+7. Counter-Forensic Discipline
+8. Physical Access Checklists
+
+---
+
+# Chapter 1: Principles of Physical Access
+
+## 1.1 The Access Spectrum
+
+Physical access operations range from the trivial to the extraordinarily complex. The skill is in choosing the right approach for the target, not in always using the most sophisticated technique.
+
+**The spectrum:**
+- **Social engineering** — Talking your way in. Lowest technical skill, highest human skill. Works against most targets because people are the weakest link in any security system.
+- **Opportunistic access** — Exploiting unlocked doors, tailgating, maintenance windows. Requires patience and observation, not tools.
+- **Technical bypass** — Defeating locks, alarms, cameras, and access control systems. Requires tools, training, and practice.
+- **Forced entry** — Breaking in. Last resort. Leaves evidence. Appropriate only when covert access is not required or not possible.
+
+The general rule: try the simplest approach first. A $5 clipboard and a confident walk will get you through more doors than a $500 lock pick set.
+
+## 1.2 The Three Requirements
+
+Every physical access operation requires three things:
+
+1. **Access** — A way to get inside the target space
+2. **Time** — Enough uninterrupted time to accomplish the objective
+3. **Cover** — A plausible explanation for being there if discovered
+
+If any of these three is missing, the operation should not proceed.
+
+---
+
+# Chapter 2: Target Assessment
+
+## 2.1 Pre-Operational Surveillance
+
+Before any physical access attempt, the target must be thoroughly assessed. This is not optional. Skipping assessment is the primary cause of physical access failures.
+
+**What to document:**
+
+**The physical space:**
+- All entry and exit points (doors, windows, loading docks, emergency exits, roof access)
+- Lock types on each entry point (pin tumbler, electronic, magnetic, padlock)
+- Alarm system type and configuration (motion sensors, door contacts, glass break, keypads)
+- Camera locations, coverage angles, and recording method (local DVR, cloud, monitored)
+- Lighting conditions (interior and exterior, timers, motion-activated)
+- Neighboring structures and sight lines
+
+**The human element:**
+- Guard schedules (shift changes, patrol patterns, break times)
+- Employee arrival and departure patterns
+- Cleaning crew schedule and access level
+- Delivery schedules (when doors are propped open, when loading docks are active)
+- Neighbor patterns (who might observe your entry)
+
+**The timing:**
+- When is the target space empty?
+- When are guards least alert? (typically 2-4 AM, immediately after shift change)
+- What events create access opportunities? (maintenance windows, construction, deliveries)
+- How long can you be inside without discovery?
+
+## 2.2 Security System Assessment
+
+**Alarm systems:**
+- Panel location (usually near main entry)
+- Entry delay zones (which doors give you time to disarm)
+- Immediate alarm zones (which doors trigger instantly)
+- Monitoring method (central station, local siren only, or unmonitored)
+- Backup power (battery duration after power cut)
+- Cellular backup (does it phone home if landline is cut?)
+
+**Access control:**
+- Credential type (key card, fob, PIN, biometric, combination)
+- Can credentials be cloned? (HID Prox cards are trivially cloneable)
+- Fail-safe vs fail-secure locks (fail-safe opens on power loss, fail-secure stays locked)
+- Anti-passback features (must badge in before badging out)
+
+**Cameras:**
+- Real vs dummy cameras (look for indicator lights, cable connections, lens quality)
+- Recording retention period (most systems overwrite after 7-30 days)
+- Monitored in real-time or reviewed only after incidents?
+- Night vision capability
+- Blind spots in coverage
+
+---
+
+# Chapter 3: Lock Bypass Techniques
+
+## 3.1 Pin Tumbler Locks (Standard Door Locks)
+
+The most common lock type in residential and commercial settings. Understanding pin tumbler mechanics is fundamental to physical access tradecraft.
+
+**How they work:** A cylinder contains spring-loaded pins of varying heights. The correct key pushes each pin to the shear line — the boundary between the plug (rotating part) and the housing (fixed part). When all pins are at the shear line, the plug rotates and the lock opens.
+
+**Single Pin Picking (SPP):**
+1. Insert tension wrench at bottom of keyway, apply light rotational pressure
+2. Insert pick above tension wrench
+3. Feel for the binding pin (the pin with the most resistance)
+4. Push binding pin to shear line — you will feel a slight click and the plug will rotate fractionally
+5. Find the next binding pin and repeat
+6. Continue until all pins are set and the plug rotates fully
+
+**Difficulty factors:** Number of pins (typically 5-6), pin tolerances (tighter = harder), security pins (spool, serrated, mushroom — provide false sets).
+
+**Time estimate:** 30 seconds to 5 minutes for a standard 5-pin lock. High-security locks may take 15-30 minutes or be effectively unpickable.
+
+**Raking:**
+A faster but less reliable method. Insert a rake (a pick with a wavy profile) and rapidly move it in and out while applying tension. This randomly sets pins through manipulation. Best for low-security locks when speed matters more than stealth.
+
+**Bump keys:**
+A specially cut key that, when struck with a bump hammer, momentarily bounces all pins above the shear line simultaneously. Simple to execute, works on most standard pin tumbler locks. Defeated by security pins and anti-bump features.
+
+## 3.2 Other Lock Types
+
+**Padlocks — Shimming:**
+Insert a thin metal shim (cut from a beverage can) between the shackle and the lock body. The shim pushes the locking pawl aside, releasing the shackle. Works on spring-loaded padlocks. Does NOT work on padlocks with anti-shim features (ball bearing locking mechanism).
+
+**Tubular locks (vending machines, utility panels):**
+Defeated with a tubular lock pick — a circular tool that simultaneously manipulates all pins. Cheap, fast, minimal skill required.
+
+**Electronic/magnetic locks:**
+- Proximity cards (HID Prox, iClass): can be cloned with $50 reader/writer
+- Magnetic stripe: trivially cloned
+- Smart cards with encryption: harder but not impossible
+- Biometric: requires different approach (replay attack, fake fingerprint, or bypass the controller rather than the sensor)
+
+**Impressioning:**
+Insert a blank key, apply turning pressure, examine marks left on the blank by the pins, file the blank at mark locations. Repeat until the blank becomes a working key. Time-intensive (30-60 minutes) but produces a working key that can be reused.
+
+## 3.3 Bypass Tools
+
+**Over-the-door tool:** A flexible tool that reaches around the door edge to manipulate the interior handle or thumb turn. Works on many commercial doors where the interior handle is unlocked.
+
+**Latch slipping:** Using a flexible card or shim to push back the spring bolt on a door. Only works on spring bolts (not deadbolts) and doors without strike plate guards.
+
+**Under-door tool:** A rigid tool slid under the door to pull the interior handle down. Effective on lever-style handles with sufficient gap under the door.
+
+## 3.4 Countermeasures to Know
+
+Locks that resist most bypass techniques:
+- **Medeco** — Rotating pins, sidebar mechanism. Extremely difficult to pick.
+- **Abloy** — Disc detainer mechanism, no springs. Requires specialized tools.
+- **Mul-T-Lock** — Pin-within-a-pin design. Doubles the effective pin count.
+- **Electronic deadbolts** — No mechanical keyway to attack.
+
+---
+
+# Chapter 4: Social Engineering for Access
+
+## 4.1 The Fundamental Principle
+
+People want to be helpful. People defer to authority. People avoid confrontation. Social engineering exploits these tendencies to bypass physical security controls that cannot be bypassed technically.
+
+A confident person in the right uniform with the right clipboard can walk into almost any building in the world.
+
+## 4.2 Effective Pretexts
+
+**IT Technician:**
+- Props: Laptop bag, cable tester, badge on lanyard (any badge — people rarely read them)
+- Script: "I'm here to check the network drop in [specific room]. We've been getting intermittent connectivity issues on this floor."
+- Why it works: IT issues are common, nobody wants to be responsible for blocking the fix, and few people understand IT well enough to question the request.
+
+**Fire Inspector / Building Inspector:**
+- Props: Clipboard, camera, measuring tape, hi-vis vest
+- Script: "We're doing the annual fire safety inspection. I need to check the exits and extinguisher placement on this floor."
+- Why it works: Inspections are expected and periodic. Building code compliance is everyone's responsibility. Nobody wants to obstruct an inspector.
+
+**Delivery Driver:**
+- Props: Uniform, hand truck, packages (can be empty boxes with printed labels)
+- Script: "Delivery for [name from LinkedIn/company directory]. Can you sign?"
+- Why it works: Deliveries happen constantly. The recipient's name adds credibility. People sign without thinking.
+
+**Contractor / Maintenance:**
+- Props: Tool bag, hard hat, hi-vis vest, work boots
+- Script: "Facilities called us about the [HVAC/plumbing/electrical] issue on the third floor."
+- Why it works: Maintenance is constant in commercial buildings. Workers go everywhere.
+
+**New Employee:**
+- Props: Business casual attire, slightly confused expression
+- Script: "Hi, I just started in [department]. I think I got turned around — can you show me where [room] is?"
+- Why it works: People empathize with being new. They want to help. They'll escort you past security checkpoints.
+
+## 4.3 Tailgating
+
+Following an authorized person through a secured door before it closes. The simplest and most effective physical access technique.
+
+**When it works best:**
+- Shift change (many people entering at once)
+- Lunch hour (heavy traffic in and out)
+- When someone is carrying items (they can't hold the door AND challenge you)
+- Loading docks (constant traffic, casual atmosphere)
+
+**Counter the challenge:** If someone asks "can I see your badge?" — pat your pockets, look frustrated, say "I must have left it at my desk. I'm [name], I work with [department]." Most people will let you through rather than make a scene. If they insist, leave gracefully and try another entry point.
+
+## 4.4 Authority Exploitation
+
+People obey authority figures reflexively. This can be exploited by projecting authority through appearance, behavior, and language.
+
+**Authority indicators:**
+- Suit and tie (or appropriate professional attire)
+- Confident, purposeful walk
+- Giving directions rather than asking for permission
+- Name-dropping specific internal contacts
+- Using internal jargon and acronyms
+- Looking busy and slightly annoyed (important people are always in a hurry)
+
+**The key insight:** Nobody challenges the person who acts like they belong. Security guards challenge people who look uncertain, who hesitate, who make eye contact and then look away. Walk in like you own the building and most people will assume you do.
+
+---
+
+# Chapter 5: Surreptitious Entry Operations
+
+## 5.1 Planning
+
+A surreptitious entry — also known as a "black bag job" — is a covert entry to a target location for intelligence collection without leaving evidence of intrusion. This is the most complex physical access operation and requires thorough planning.
+
+**Team composition:**
+- **Entry specialist** — handles lock bypass and alarm neutralization
+- **Inside operator** — conducts the search, photographs documents, plants devices
+- **Lookout/communications** — monitors the perimeter, provides early warning
+- **Driver** — manages transport, maintains escape vehicle
+
+Minimum team: 2 (entry + lookout). Optimal: 3-4.
+
+**Pre-operation requirements:**
+1. Complete target assessment (Chapter 2)
+2. Guard schedule confirmed through multi-day observation
+3. Alarm system identified and bypass method determined
+4. Entry point selected and lock bypass method practiced
+5. Communication plan established (radios, signals, check-in schedule)
+6. Time limit defined (30-60 minutes maximum for most operations)
+7. Abort criteria defined (what triggers immediate withdrawal)
+8. Escape routes planned (minimum 2 routes from target)
+
+## 5.2 Execution Procedures
+
+1. **Pre-entry:** Lookout confirms area is clear. Driver positions escape vehicle. Team approaches via planned route.
+2. **Entry:** Bypass lock. Neutralize alarm (if present). Confirm interior is clear.
+3. **Document existing state:** Before touching anything, photograph the room. Note the position of objects, papers, doors, drawers. Everything must be returned to its exact original position.
+4. **Conduct operation:** Search systematically. Photograph documents in place. If planting a device, select a location that will not be disturbed.
+5. **Time checks:** Lookout provides regular time checks. Operations cease at the pre-defined time limit regardless of completion status.
+6. **Exit preparation:** Verify everything is returned to original position. Check for evidence of entry (footprints, disturbed dust, moved objects).
+7. **Exit:** Re-secure lock. Restore alarm. Depart via planned route.
+8. **Post-operation:** Separate departure by team members. Counter-surveillance on departure route. No post-operation communication for defined period.
+
+## 5.3 Evidence Discipline
+
+The entire point of surreptitious entry is that the target never knows it happened. This requires fanatical attention to evidence.
+
+- Wear gloves at all times (latex underneath, nitrile on top — double layer reduces print risk and tearing)
+- Wear shoe covers or shoes purchased for this operation only
+- Do not eat, drink, smoke, or spit inside the target space (DNA)
+- Photograph everything before touching it
+- Replace every object in its exact position (use the photographs as reference)
+- Check for hair, fibers, or other trace evidence before departing
+- If you move furniture or equipment, note its exact position and return it precisely
+
+---
+
+# Chapter 6: Postal & Logistics Operations
+
+## 6.1 The Postal Channel
+
+Physical mail remains one of the most effective channels for covert material transfer. It is anonymous (no ID required to mail a package), ubiquitous (billions of packages annually), and relatively uninspected (inspection rates are very low for domestic mail).
+
+However, postal inspection services have developed profiling criteria to identify suspicious packages. Understanding and avoiding these triggers is essential.
+
+## 6.2 FBI/USPS Profiling Criteria
+
+From leaked inspection guidelines (analyzed by the Grugq):
+
+**Packaging red flags:**
+- Heavy taping along seams
+- Package reuse (old labels, previous shipping marks)
+- Uneven weight distribution
+- Poor preparation for mailing
+
+**Labeling red flags:**
+- Handwritten labels (most business mail uses printed labels)
+- Misspellings
+- Person-to-person (not business-to-individual)
+- Return zip code doesn't match the accepting post office zip code
+- Fictitious return address
+- Sender/recipient names that are obviously generic (John Smith)
+- No connection between sender name and return address
+
+**Content indicators:**
+- Weight approximately equal to round metric amounts (1 kg + packaging)
+- Express Mail for non-document items (Express Mail is primarily used for business document delivery)
+- Origination from known source locations
+
+## 6.3 Countermeasures (Avoiding Profiling)
+
+**Packaging:**
+- Use new, professional packaging (not reused boxes)
+- Minimal, clean taping — enough to secure, not suspicious
+- Use appropriate box size for contents (no excessive padding)
+- Package should look like normal commercial mail
+
+**Labeling:**
+- Print labels (never handwrite)
+- Use a business-to-individual format
+- Include a plausible business name as sender
+- Return address must be real and verifiable (backstopped)
+- Return zip code must match the post office where you mail it
+- Recipient name should match someone at the delivery address
+
+**Backstopping:**
+- The return address must correspond to a real location
+- The sender name should match someone who could plausibly be at that address
+- If checked, the address should not raise immediate flags
+- Research the identity — don't invent one. Borrow one.
+
+As the Grugq notes: "Don't make shit up, do your research and steal an identity with a real address. The complexity and depth of that backstop are dependent on how deeply the cover will be investigated."
+
+---
+
+# Chapter 7: Counter-Forensic Discipline
+
+## 7.1 Tamper-Evident Awareness
+
+Many targets employ passive tamper detection — indicators that reveal whether a space has been accessed:
+
+- **Hair or thread across door crack** — Falls when door is opened. Check before entry.
+- **Dust patterns** — Undisturbed dust on surfaces indicates no recent access. Your presence will disturb it.
+- **Tamper-evident seals** — Adhesive seals on containers, cabinets, or doors that show signs of removal.
+- **Powder traps** — Fine powder on floors or surfaces that captures footprints.
+- **Hidden cameras** — Motion-activated cameras in non-obvious locations (clocks, smoke detectors, power outlets).
+- **Object positioning** — Items deliberately placed to detect movement (paper at specific angle, pen position on desk).
+
+## 7.2 Digital Forensic Awareness
+
+If the operation involves accessing a computer:
+- Do NOT boot the target machine (creates forensic artifacts)
+- If the machine is already on, do NOT log in without understanding what logging is enabled
+- Photograph the screen before touching anything
+- Use forensic tools that minimize artifacts (live USB, hardware imager)
+- If planting a device, ensure it does not create detectable artifacts in system logs
+
+---
+
+# Chapter 8: Physical Access Checklists
+
+## 8.1 Social Engineering Checklist
+
+- [ ] Pretext selected and rehearsed
+- [ ] Props acquired (uniform, badge, clipboard, tools as appropriate)
+- [ ] Target contact names researched (for name-dropping)
+- [ ] Internal jargon researched
+- [ ] Entry time selected (shift change, delivery hours, lunch)
+- [ ] Backup pretext prepared
+- [ ] Exit strategy defined
+- [ ] Communications plan established
+
+## 8.2 Surreptitious Entry Checklist
+
+- [ ] Target assessment complete (layout, security, schedule)
+- [ ] Multi-day observation confirms patterns
+- [ ] Entry point selected and bypass method tested
+- [ ] Alarm type identified and bypass planned
+- [ ] Team briefed and roles assigned
+- [ ] Communication equipment tested
+- [ ] Gloves, shoe covers, and evidence discipline gear ready
+- [ ] Camera for documenting original state
+- [ ] Time limit defined (30-60 minutes)
+- [ ] Abort criteria defined
+- [ ] Primary and secondary escape routes planned
+- [ ] Counter-surveillance planned for approach and departure
+- [ ] Cover story prepared if intercepted
+- [ ] Post-operation communication blackout period defined
+
+---
+
+*This manual is a living document. Content will be enriched as additional sources are collected and analyzed.*

output/manuals/06_OPSEC_Principles.md

@@ -0,0 +1,473 @@
+# OPSEC Principles Manual
+
+> Synthesized from: Allen Dulles "Some Elements of Intelligence Work," the Grugq's OPSEC analyses
+> (Silk Road, Harvard bomb hoax, Morris Worm, PIRA, Reservoir Dogs, Yardbird), JSOU Report 12-3,
+> CIA CHECKPOINT travel intelligence
+>
+> Classification: OPEN SOURCE — compiled from publicly available materials
+
+---
+
+## Table of Contents
+
+1. OPSEC Fundamentals
+2. The Dulles Foundation
+3. Compartmentation
+4. The Operational Phase Framework
+5. Identity & Persona Management
+6. Communications Security
+7. The STFU Principle
+8. Informant Awareness & Counter-Intelligence
+9. Case Studies
+10. The OPSEC Checklist
+
+---
+
+# Chapter 1: OPSEC Fundamentals
+
+## 1.1 What OPSEC Is (And What It Isn't)
+
+OPSEC is not a tool. It is not encryption software. It is not Tor. It is not a VPN. It is not a burner phone.
+
+**OPSEC is a mode of operating.**
+
+This distinction matters because people who think OPSEC is a tool believe they can install it, configure it, and forget it. They are wrong. They will be caught.
+
+OPSEC is the continuous discipline of minimizing the information available to your adversary about your identity, intentions, capabilities, and activities. It encompasses every decision you make — what you say, where you go, what you buy, who you associate with, what devices you use, what patterns you create.
+
+As the Grugq writes: "OPSEC is a mode of operating, not a tool or a collection of tools." And: "It is more important to compartment sensitive activities and structure your operational environment for impact containment than to install particular software."
+
+## 1.2 The OPSEC Tradeoff
+
+High OPSEC means low efficiency. High efficiency means weak OPSEC. There is no way around this tradeoff. Every security measure adds friction — latency in communications, complexity in logistics, constraints on behavior. The question is never "how much OPSEC do I need?" but rather "how strong is my adversary?"
+
+The strength of opposing forces dictates minimum security requirements. Operating against a corporate security team requires different OPSEC than operating against a nation-state intelligence service. Calibrate accordingly, but when in doubt, overestimate the adversary.
+
+## 1.3 The Practice Imperative
+
+"Amateurs practice until they get it right. Professionals practice until they can't get it wrong."
+
+OPSEC routines must be automatic. Under stress, you will not rise to the level of your aspirations — you will fall to the level of your training. If you have never practiced an SDR, you will not execute one properly when you actually need it. If you have never used a dead drop, your first attempt under operational pressure will be clumsy and observable.
+
+The Dulles corollary: "The man or woman who does not indulge in the daily security routine, boring and useless though it may sometimes appear, will be found lacking in the proper instinctive reaction when dealing with the bigger stuff."
+
+Practice your security routines during peacetime. Make them habitual. When the moment comes, habit will save you where conscious thought cannot.
+
+---
+
+# Chapter 2: The Dulles Foundation
+
+Allen Dulles distilled a career of intelligence work into a set of principles that have not been improved upon in seventy years. They are presented here not as historical curiosities but as operational fundamentals.
+
+## 2.1 Security Above All
+
+"The greatest of them all is security. All else must be subordinated to that."
+
+This is the meta-rule. Every other principle derives from this one. When in conflict between security and any other objective — speed, convenience, thoroughness, ego — security wins. Always.
+
+## 2.2 Security Lives in the Details
+
+"The little things are in many ways more important than the big ones. It is they which oftenest give the game away."
+
+You are more likely to be compromised by a careless phone call than by a broken cipher. You are more likely to be identified by a receipt in your pocket than by a failed dead drop. The big operational decisions get careful thought. The small daily habits are where discipline breaks down.
+
+Specific Dulles directives:
+- Never leave things unattended or where you might forget them
+- Learn to write lightly — the blank page underneath has been read
+- Destroy documents thoroughly, not casually
+- Carry as little written material as possible, for the shortest possible time
+- Never carry names or addresses in clear text — use a personal code only you understand
+- If you must clip small papers to larger ones, do so — loose papers get lost
+
+## 2.3 Never Admit
+
+"Even though you feel the curious outsider has probably a good idea that you are not what you purport to be, never admit it. Keep on playing the other part."
+
+People doubt their own suspicions. If you maintain your cover consistently, observers will often convince themselves they were wrong. But the moment you admit — even partially, even indirectly — the game is over. No half-admissions. No "well, I'm a little involved in that." Nothing.
+
+## 2.4 Vanity Is the Enemy
+
+"The next greatest vice is that of vanity. Its offshoots are multiple and malignant."
+
+The need to be recognized, to show how clever you are, to hint at your secret life — this is the operational equivalent of painting a target on your chest. The Provisional IRA lost members because they boasted in pubs. Ross Ulbricht lost Silk Road because he needed social validation on forums. Robert Morris lost his freedom because he couldn't resist explaining his worm to friends.
+
+The best operative is the one nobody suspects. If you need external validation for your work, find a different profession.
+
+## 2.5 The Telephone Problem
+
+"The greatest material curse to the profession is undoubtedly the telephone."
+
+Written in the 1950s, this warning has only become more relevant. Modern phones are tracking devices that record your location, contacts, timing, metadata, and potentially content. The telephone is not merely a temptation to slackness — it is an active surveillance device you carry voluntarily.
+
+Dulles's rule: "Always act on the principle that every conversation is listened to." This was prudent caution in 1955. In 2025, it is a documented fact.
+
+Modern application: See Chapter 6 (Communications Security) for detailed phone OPSEC procedures.
+
+## 2.6 Carelessness Is Irreversible
+
+"Mistakes made generally cannot be rectified."
+
+In most professions, a mistake is a learning experience. In operational security, a mistake is a compromise. You cannot un-send the email that contained your real name. You cannot un-visit the location that linked your cover to your identity. You cannot un-say the words that confirmed a suspicion.
+
+Think before you act. Then think again. Then act.
+
+---
+
+# Chapter 3: Compartmentation
+
+## 3.1 The Principle
+
+Compartmentation is the separation of information, people, and activities into discrete cells with no interaction, access, or knowledge of each other. It is the cornerstone of any solid counter-intelligence program.
+
+If any single compartment is compromised — by informant, surveillance, or technical penetration — the damage stops at the compartment boundary. Without compartmentation, a single compromise cascades through the entire operation.
+
+## 3.2 Types of Compartmentation
+
+**Organizational compartmentation:** Structuring a group so that cells are isolated from each other. Members of Cell A know only Cell A's members and have contact with one liaison in Cell B. That is the limit of their knowledge.
+
+**Functional compartmentation:** Separating activities so that each function (surveillance, logistics, execution, communications) is performed by a different team with no cross-visibility.
+
+**Temporal compartmentation:** Regularly changing communications platforms, identities, and infrastructure to create chronological silos. Old identities and channels are abandoned. Compromise of a current compartment cannot reach into past compartments.
+
+**Personal compartmentation:** Separating your own illicit activity from your regular life. This is what CIA case officers do — they compartment their espionage from their cover life. The first rule: never discuss your illicit activities with anyone outside the compartment.
+
+## 3.3 Compartmentation for Individuals
+
+The Grugq provides a practical framework for personal compartmentation:
+
+**The threat model:** Two people (Alice and Bob) want to exchange information. They need to protect against an adversary learning:
+1. That two people have been in contact (low risk)
+2. That Bob has been in contact with someone (medium risk)
+3. That Alice has been in contact with someone (high risk)
+4. That Alice has been in contact with Bob (extreme risk)
+
+Each risk level requires different countermeasures. Protecting against (1) requires hiding the existence of communication. Protecting against (4) requires complete persona separation.
+
+**Practical implementation:**
+- Separate devices for separate compartments (never use the same laptop for personal and operational activity)
+- Separate locations for separate compartments (never conduct operational communications from home)
+- Separate identities for separate compartments (no shared usernames, emails, or behavioral patterns)
+- Never cross-contaminate between compartments
+- If a compartment is compromised, the others survive
+
+## 3.4 The Cost of Poor Compartmentation
+
+Ross Ulbricht ran the Silk Road marketplace for over two years. He was caught because his operational persona (Dread Pirate Roberts) and his personal identity (Ross Ulbricht) shared:
+- Ideology (Austrian School of Economics, mises.org)
+- Geographic location (San Francisco)
+- Timezone (evident in posting patterns)
+- Technical interests (PHP, Bitcoin security)
+- An email address (posted from the "altoid" operational persona on BitcoinTalk)
+- Server access location (frosty@frosty.com admin accessed Silk Road server from 500 feet from Ulbricht's Gmail login location)
+
+Any ONE of these links might not have been sufficient. Together, they were damning. Compartmentation must be total or it is meaningless.
+
+---
+
+# Chapter 4: The Operational Phase Framework
+
+## 4.1 The Five Phases
+
+Every operation — cyber or physical, offensive or defensive — moves through five distinct phases:
+
+1. **Target Selection** — Choosing what to attack/collect/influence
+2. **Planning & Surveillance** — Gathering information, developing the plan
+3. **Deployment** — Moving into position, preparing infrastructure
+4. **Execution** — Conducting the operation
+5. **Escape & Evasion** — Getting away clean
+
+Most failures happen because phases 4 and 5 receive less attention than phases 1-3. The Grugq notes that "all real criminals know that the most important part of an operation is the getaway." Hackers and amateur operatives routinely neglect the escape phase.
+
+## 4.2 Phase Analysis
+
+**Target Selection:** This is where strategy meets intelligence. Poor target selection wastes resources and creates unnecessary exposure. Security forces prioritize identifying people involved in target selection and planning — they are the principals, more valuable than the people who execute.
+
+**Planning & Surveillance:** The phase where most intelligence collection occurs. Pattern of life analysis, vulnerability assessment, route planning. Also the phase where pre-operational contact creates the most risk. Every meeting, every communication, every reconnaissance visit creates a potential link.
+
+**Deployment:** Moving from planning to readiness. Infrastructure is activated, teams are positioned, logistics are finalized. This is the point of commitment — once deployment begins, the operation is live.
+
+**Execution:** The operation itself. Duration should be minimized. The longer you are exposed, the greater the risk. Execute the plan, don't improvise unless the plan fails.
+
+**Escape & Evasion:** The most neglected and most critical phase. How do you leave the scene? How do you dispose of evidence? How do you break the link between yourself and the operation? How do you return to normal life without anomalous behavior?
+
+## 4.3 The Harvard Bomb Hoax — Phase Failure Analysis
+
+In December 2013, Harvard student Eldo Kim sent bomb threats to avoid a final exam. His operational analysis through the phase framework:
+
+**Target Selection:** Correct — buildings where his exam was held.
+**Planning:** Minimal — chose Tor Browser Bundle and GuerillaMail.
+**Deployment:** Fatal error — used Tor from the Harvard campus network.
+**Execution:** Succeeded — emails were sent.
+**Escape & Evasion:** Non-existent — no plan for what happens after.
+
+The deployment error was decisive. Using Tor from the campus network reduced the anonymity set from "anyone on the internet" to "Harvard students using Tor at the time the bomb threats were sent." That was a very small number. He was identified within hours.
+
+The lesson: **Never take an action that reduces the pool of suspects.** If all students are suspects, you only need to avoid narrowing the pool. Using an anonymizing tool from a specific location during a specific time window does exactly the opposite.
+
+---
+
+# Chapter 5: Identity & Persona Management
+
+## 5.1 The Contamination Problem
+
+Contamination occurs when information from one persona leaks into another. It is the most common cause of identity compromise, and it is almost always caused by convenience or carelessness, not by technical failure.
+
+**Types of contamination:**
+- **Direct link:** Using a personal email from an operational account (Ulbricht)
+- **Behavioral link:** Both personas share ideology, writing style, or interests (DPR/Ulbricht)
+- **Geographic link:** Accessing both personas from the same location (Ulbricht)
+- **Temporal link:** Both personas are active at the same times, inactive at the same times
+- **Technical link:** Same device, same browser fingerprint, same IP, same VPN exit node
+
+## 5.2 Persona Separation Rules
+
+1. **Separate devices.** Never use the same physical device for different personas. If this is impossible, use separate VMs with separate network paths.
+2. **Separate locations.** Never access different personas from the same physical location. This includes WiFi networks — your home WiFi MAC address is unique.
+3. **Separate behaviors.** Different personas should have different writing styles, different interests, different posting schedules, different political views.
+4. **Separate infrastructure.** Different email providers, different VPNs, different cryptocurrency wallets, different SIM cards.
+5. **Never cross-contaminate.** Never copy-paste between persona environments. Never visit a personal site from an operational browser. Never use operational tools from a personal device.
+
+## 5.3 Isolation and Its Dangers
+
+Underground operatives face a severe psychological challenge: isolation. The security requirements of deep cover preclude normal social interaction. Ulbricht's case illustrates what happens:
+
+- He rented a room under an assumed name
+- He had no "mainstream" social circle to calibrate against
+- His only social interaction was with Silk Road forum members and admins
+- Social isolation drove him toward ideological extremism
+- Isolation degraded his security discipline over time
+
+**Mitigation:** Maintain a cover social life that provides genuine human connection. Have non-operational friends. Exercise. Maintain routines that keep you grounded in normal reality. The underground life, sustained too long in isolation, produces bad judgment.
+
+---
+
+# Chapter 6: Communications Security
+
+## 6.1 The Four Goals
+
+Secure communications must achieve four objectives, in ascending order of difficulty:
+
+1. **Content protection** — Make the message unreadable to unauthorized parties. Solved by encryption.
+2. **Meaning protection** — Make the message's significance inaccessible even if the text is readable. Addressed by codes.
+3. **Traffic analysis resistance** — Prevent the adversary from knowing that a connection exists between the communicating parties. Very difficult.
+4. **Channel concealment** — Prevent the adversary from knowing that the communication channel exists at all. Extremely difficult.
+
+Most people stop at goal 1. Encryption is the easy part. Goals 3 and 4 are where operations succeed or fail.
+
+## 6.2 Metadata Kills
+
+The content of your message matters less than the fact that you sent it. Modern intelligence services collect metadata — who communicated with whom, when, for how long, from where — at massive scale. Metadata reveals:
+- Organizational structure (who talks to whom)
+- Operational tempo (communication frequency increases before operations)
+- Geographic patterns (where calls originate)
+- Relationships (contact frequency indicates relationship strength)
+
+**Encryption protects content. It does not protect metadata.** The CIA Lebanon rollup demonstrates this: Hezbollah identified CIA agents not by breaking encryption but by identifying phones that were used exclusively for handler communication. The pattern — dedicated device, static location, predictable activation schedule — was the vulnerability.
+
+## 6.3 The Telephone Rules (Modern Application)
+
+Dulles warned about the telephone in the 1950s. The Grugq updated these warnings for the mobile era:
+
+**Physical identifiers:**
+- IMEI — unique hardware identifier for the phone device itself
+- IMSI — unique subscriber identifier on the SIM card
+- Both must be changed together to break the link
+
+**Location tracking:**
+- 4 location data points will uniquely identify 90% of mobile phone users
+- Your mobility pattern (home → commute → work → gym) is as unique as a fingerprint
+- "Mirroring" — when two devices travel together — links them permanently
+
+**Burner phone rules:**
+1. Phone OFF means battery removed, SIM removed, placed in shielded bag
+2. Never power on at locations associated with you — home, work, friends
+3. Never turn on your burner at the same location as your personal phone
+4. Never let your personal phone go OFF when your burner goes ON (paired events are indicators of relation)
+5. Never carry phones for different compartments together
+6. Store the burner away from your home
+7. Keep your personal phone showing normal usage pattern at all times
+8. Buy with cash, never register, discard after limited use
+
+## 6.4 Codes vs. Encryption
+
+Codes protect meaning. Encryption protects content. They serve different purposes.
+
+The US Army COMSEC handbook warns against "talking around" — trying to discuss sensitive subjects using circumlocution. "Self-made reference systems" rarely work because "few people are clever enough to refer to an item of information without actually revealing names, subjects, or other pertinent information."
+
+**Practical code discipline:**
+- Keep codes generic and consistent
+- Limit codes to simple signaling (go/no-go, danger/safe, meeting confirmed/canceled)
+- Do not attempt to discuss complex operational details via code
+- Pre-arrange all codes before the operation begins
+- Use different code systems for different compartments
+
+---
+
+# Chapter 7: The STFU Principle
+
+## 7.1 The Rule
+
+If someone is not actively sharing the risk of an operation, they have no need to know about it. Period.
+
+This is the single most violated principle in operational security. People talk. They talk because they are proud, because they are scared, because they are drunk, because they want to impress someone, because they need to process what they've experienced. Every word spoken to someone outside the operation is a potential compromise.
+
+## 7.2 The Morris Worm — Case Study in Talking
+
+Robert Morris created the first major internet worm in 1988. He was brilliant. He was also incapable of keeping quiet.
+
+Morris briefed his friends on all aspects of the worm: how it was developed, how it worked, what vulnerabilities it exploited. At one meeting at a Legal Seafood restaurant, he was so excited that "he literally jumped up on a table pacing back and forth on the table explaining how it worked."
+
+His friends were subpoenaed. They testified. Morris was convicted.
+
+His lawyer later reflected: "He did testify that he wrote the worm. He came in and testified, 'I did it, and I'm sorry.' I turned to my co-counsel and asked, 'Should I prove he didn't do it or he's not sorry?'"
+
+## 7.3 Need-to-Know Evaluation
+
+Before sharing any operational information, ask:
+1. Is this person actively sharing the risk of this operation?
+2. Do they need this specific piece of information to perform their role?
+3. Will withholding this information degrade their ability to function?
+
+If the answer to any of these questions is no, do not share. Even within the operation, restrict knowledge to the specific aspects each person needs. The team driver does not need to know the exfiltration plan. The lookout does not need to know the target's name.
+
+---
+
+# Chapter 8: Informant Awareness
+
+## 8.1 The Oldest Threat
+
+Technical surveillance can be defeated with tradecraft. Cryptography can protect communications. Compartmentation can contain damage. But none of these measures protect against an informant — a member of your own organization who reports to the adversary.
+
+When a group with robust security practices is compromised and the technical indicators don't explain how, the answer is almost always an informant.
+
+## 8.2 The PIRA Lesson
+
+The Provisional IRA in the 1970s demonstrated every form of self-incrimination possible:
+- Singing IRA songs in pubs (public affiliation)
+- Boasting about operations while drunk (direct disclosure)
+- Responding to inquiries with "a nod and a wink" (confirmation)
+- Attending pro-IRA rallies (surveillance opportunity)
+- Socializing with operational colleagues off-duty (pre-operational contact)
+
+British security forces exploited this systematically. Known members (identified through public behavior) were monitored. Their social contacts were mapped. Unknown members were identified through association. The entire organizational graph was eventually exposed through link analysis starting from a handful of pub boasters.
+
+**The lesson is simple:** Never publicly display operational affiliation. Never socialize with operational contacts in personal contexts. Never discuss operations outside of operational necessity. Alcohol is a vulnerability, not a social lubricant.
+
+## 8.3 Detection Indicators
+
+Signs that an organization may have been penetrated:
+- Opposition demonstrates knowledge of operations that should be compartmented
+- Arrests or disruptions that don't match the opposition's known technical capabilities
+- Indictments with unusually specific geographic information about members
+- Operations fail in ways that suggest foreknowledge
+- "Lucky" coincidences that benefit the opposition repeatedly
+
+The Grugq's analysis of the Lauri Love hacking case: "The lack of information on how Mr Love was caught, along with the revelation of good security practices suggests one thing: informant."
+
+## 8.4 Anti-Informant Measures
+
+From the Reservoir Dogs SOP (Fatah/BSO methodology):
+1. **Assigned aliases** — Random, per-operation, prevents pattern development
+2. **Just-in-time assembly** — Form team immediately before operation
+3. **Dedicated support teams** — Each function compartmented
+4. **Strict need-to-know** — No one knows the complete plan except the principal
+5. **Post-operation dispersal** — Team disbanded, infrastructure destroyed
+
+These measures limit the damage an informant can cause but do not prevent infiltration. The only true defense against informants is rigorous vetting and the acceptance that some level of penetration risk is inherent in any organization with human members.
+
+---
+
+# Chapter 9: Case Studies
+
+## 9.1 Silk Road — The Complete Failure
+
+**Timeline:** 2011-2013
+**Actor:** Ross Ulbricht (Dread Pirate Roberts)
+**Outcome:** Arrested, convicted, life sentence
+
+**What went right:**
+- Tor hidden service for the marketplace
+- Bitcoin for financial transactions
+- Pseudonymous identity (DPR)
+
+**What went wrong (everything else):**
+1. Used personal email from operational persona (altoid → rossulbricht@gmail.com)
+2. Shared ideology between personas (Austrian economics on both accounts)
+3. Same geographic location (San Francisco) and timezone
+4. Server admin access from same physical location as personal Gmail
+5. Social isolation drove extremism and poor judgment
+6. Sought social validation on forums, lowering security discipline
+7. No backstopping of operational personas
+8. Used same computer for personal and operational activity
+
+**Key lesson:** Compartmentation is binary. It either exists or it doesn't. A single contamination link between personas makes all other security measures irrelevant.
+
+## 9.2 Harvard Bomb Hoax — Anonymity Set Reduction
+
+**Timeline:** December 2013
+**Actor:** Eldo Kim (Harvard student)
+**Outcome:** Identified within hours
+
+**The operation:** Send bomb threats to cancel a final exam.
+**The method:** Tor Browser Bundle + GuerillaMail from Harvard campus.
+**The failure:** Using Tor from the campus network reduced the suspect pool to Harvard Tor users during the threat window — a very small number.
+
+**Key lesson:** Anonymity tools provide anonymity only within the set of users at that location and time. Smaller the set, less the anonymity. He should have used a nearby cafe with no connection to the university.
+
+## 9.3 CIA Lebanon — Phone Pattern Analysis
+
+**Timeline:** 2011
+**Actor:** CIA spy network
+**Adversary:** Hezbollah counter-intelligence
+**Outcome:** Entire network rolled up
+
+**The tradecraft:** Agents used dedicated mobile phones for handler communication. Phones kept at static locations. Pre-arranged meeting at fixed location.
+
+**The failure:** Hezbollah identified the pattern — dedicated devices that activated only for specific contacts, from specific locations, at predictable intervals. The pattern, not the content, was the vulnerability.
+
+**Key lesson:** Encryption protects content. Anonymity protects identity. Systems based on secrecy alone — where usage itself is anomalous — attract attention and enable pattern analysis. Anonymity must come before encryption.
+
+---
+
+# Chapter 10: The OPSEC Checklist
+
+## 10.1 Pre-Operation
+
+- [ ] Threat model defined (who is the adversary, what are their capabilities?)
+- [ ] OPSEC level calibrated to adversary strength
+- [ ] Operational compartmentation established
+- [ ] Separate devices for separate compartments
+- [ ] Separate identities with no cross-contamination
+- [ ] Communication plan established (methods, schedules, emergency protocols)
+- [ ] Cover story developed and backstopped (if applicable)
+- [ ] Need-to-know enforced for all participants
+- [ ] SDR procedures planned (if physical component)
+- [ ] Escape and evasion plan developed
+- [ ] Evidence disposal plan developed
+- [ ] Emergency abort criteria defined
+
+## 10.2 During Operation
+
+- [ ] SDR executed before any operational activity
+- [ ] Cover maintained at all times
+- [ ] Communications limited to operational necessity
+- [ ] No unnecessary logs or records created
+- [ ] Devices used only for designated compartment
+- [ ] No pre-operational contact with team members outside operational context
+- [ ] Counter-surveillance awareness maintained
+- [ ] Anomalies noted and assessed
+
+## 10.3 Post-Operation
+
+- [ ] Operational infrastructure sanitized or destroyed
+- [ ] Temporary identities retired
+- [ ] Devices wiped or physically destroyed
+- [ ] No post-operational contact with team members
+- [ ] No discussion of operation with anyone outside need-to-know
+- [ ] Return to normal behavioral patterns without anomalies
+- [ ] After-action review conducted (lessons learned, security assessment)
+- [ ] STFU
+
+---
+
+*This manual is a living document. Content will be enriched as additional sources are collected and analyzed.*

output/manuals/07_Cyber_Implants_Tools.md

@@ -0,0 +1,391 @@
+# Cyber Implants & Tools Manual
+
+> Synthesized from: WikiLeaks Vault 7 (CIA Hacking Tools), Vault 8 (CIA Source Code),
+> CIA Engineering Development Group documentation, classified tool user guides and design documents
+>
+> Classification: Source material is SECRET//NOFORN (leaked). This synthesis is for
+> educational/red team reference purposes.
+
+---
+
+## Table of Contents
+
+1. Overview — CIA Cyber Arsenal
+2. Implants & Backdoors
+3. Persistence Frameworks
+4. Network Device Exploitation
+5. Air-Gap Operations
+6. Credential Theft
+7. Collection Tools
+8. Support & Evasion Tools
+9. False Flag Operations (Umbrage)
+10. Operational Patterns & Detection
+
+---
+
+# Chapter 1: Overview — CIA Cyber Arsenal
+
+## 1.1 The Engineering Development Group
+
+The majority of Vault 7 tools were developed by the CIA's Engineering Development Group (EDG), a unit within the Directorate of Digital Innovation. EDG functions as the CIA's internal offensive tool development shop — the equivalent of NSA's TAO (Tailored Access Operations) but focused on human-deployed tools rather than network-based exploitation.
+
+Tools are classified SECRET//NOFORN and follow a formal development lifecycle with requirements documents, user guides, test plans, Independent Verification & Validation Review (IVVRR) checklists, and Tool Delivery Reviews (TDR). This bureaucratic rigor means the tools are well-documented — and that documentation is now public.
+
+## 1.2 Tool Design Philosophy
+
+Analyzing the Vault 7 tools reveals consistent design patterns:
+
+**Platform coverage:** Windows is the primary target (Athena, Angelfire, AfterMidnight, Pandemic, Dumbo, EzCheese, Brutal Kangaroo). Linux has dedicated tools (OutlawCountry, Aeris, Gyrfalcon). macOS has limited coverage (Achilles). Mobile platforms have specialized tools (HighRise for Android). Network devices are targeted with firmware implants (CherryBlossom).
+
+**Persistence over capability:** Most tools prioritize survival — staying on the target system undetected — over offensive capability. The persistence mechanisms are sophisticated (boot sector modification, DLL hijacking, kernel modules, covert filesystems). The actual collection/exfiltration capabilities are often straightforward once persistence is achieved.
+
+**Modularity:** Tools like AfterMidnight use a modular architecture where the persistence mechanism (the "platform") is separate from the capability modules ("Gremlins"). This allows operators to deploy the platform once and load/unload capabilities as needed without re-deploying the implant.
+
+**Minimal footprint:** Tools are designed to minimize their forensic footprint. BadMFS creates a covert filesystem invisible to the OS. Marble obfuscates strings to defeat forensic analysis. Dumbo disables recording devices during physical access operations.
+
+---
+
+# Chapter 2: Implants & Backdoors
+
+## 2.1 Athena (Windows)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG + Siege Technologies (contractor)
+**Platforms:** Windows 7, 8, 10
+**MITRE ATT&CK:** T1574.001 (DLL Search Order Hijacking), T1543.003 (Windows Service)
+
+**What it is:** A beacon-loader implant that runs in user space and communicates from within the srvhost process. At its core, Athena is deceptively simple — it hijacks a system DLL, loads into a legitimate service process, and beacons to a command-and-control server.
+
+**Two variants:**
+
+**Athena-Alpha** targets the RemoteAccess service. This service enumerates the registry to find an IP support DLL called `iprtrmgr.dll`. The implant replaces this DLL and forwards all original export functions to the legitimate module. When the service starts, the implant loads alongside it. By default, the RemoteAccess service is disabled — the installer enables it.
+
+**Athena-Bravo** targets the Dnscache service. This service looks for a support DLL called `dnsext.dll`. This extension is available on Windows 7 and 8 (not legacy OS). By default, this service is active, making Bravo less conspicuous than Alpha (no need to enable a disabled service).
+
+**Persistence mechanism — DLL Forwarding:**
+The target DLLs export a small number of functions. The implant DLL forwards those function calls to the original DLL at startup time. The service continues to function normally while the implant runs in the background. This is invisible to the user and to most security software because the service behaves exactly as expected.
+
+**Key technical detail:** After installation, the srvhost process must be updated to allow the implanted service to run as SYSTEM. The installer modifies the srvhost list to include the service in a SYSTEM srvhost with correct privileges. Until a reboot, the tool has limited security access.
+
+**Detection indicators:**
+- Unexpected `iprtrmgr.dll` or `dnsext.dll` in system directories with non-Microsoft signatures
+- RemoteAccess service enabled on systems where it should be disabled
+- Modified srvhost service grouping
+- Beacon traffic from srvhost process to unknown external IPs
+- DLL forwarding chains (imports that redirect to other DLLs)
+
+## 2.2 Pandemic (Windows)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Version:** 1.1 (January 2015)
+**Platforms:** Windows (SMB file servers)
+**MITRE ATT&CK:** T1080 (Taint Shared Content), T1221 (Template Injection)
+
+**What it is:** A file system minifilter driver that replaces files on-the-fly as remote users access them via SMB. When a target system running Pandemic serves files to a remote client, the files are replaced with trojanized versions during transit. The original files on disk are never modified.
+
+**How it works:** Pandemic installs as a Windows filesystem minifilter driver — the same type of driver used by antivirus software to intercept file operations. When a remote user requests a file via SMB, the minifilter intercepts the read operation and substitutes a different file. The original file remains untouched on disk. Only the remote user receives the modified version.
+
+**Operational significance:** This is a lateral movement and distribution tool. A single Pandemic installation on a file server can distribute trojanized software to every user who accesses that server. The infection spreads as users execute the files they download.
+
+**Detection indicators:**
+- Unexpected minifilter driver registered in the filter manager stack
+- Files served via SMB that differ from their on-disk copies (compare network capture vs disk forensics)
+- Minifilter driver with no corresponding legitimate software installation
+
+## 2.3 OutlawCountry (Linux)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Version:** 1.0 (June 2015)
+**Platforms:** Linux (CentOS 6.x, 64-bit with kernel 2.6.32)
+**MITRE ATT&CK:** T1014 (Rootkit), T1599 (Network Boundary Bridging)
+
+**What it is:** A Linux kernel module that provides covert netfilter-based internet traffic redirection. Once loaded, it creates hidden iptables rules that are invisible to user-space tools (`iptables -L` does not show them). This allows the operator to redirect network traffic through controlled infrastructure without the system administrator's knowledge.
+
+**How it works:** OutlawCountry loads as a standard kernel module but creates a hidden netfilter table that is not visible through normal iptables interfaces. The operator can add rules to this hidden table to redirect outbound traffic (DNS, HTTP, HTTPS) to CIA-controlled servers for interception or modification.
+
+**Operational significance:** This is a persistent network interception tool. On a Linux server that routes traffic for an organization, OutlawCountry can silently redirect all traffic through CIA infrastructure.
+
+**Detection indicators:**
+- Unexpected kernel modules loaded (`lsmod`, `/proc/modules`)
+- Netfilter tables that don't match the visible iptables configuration
+- Network traffic taking unexpected routes (traceroute anomalies)
+- Kernel module files in unexpected locations
+
+## 2.4 Aeris (POSIX)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Linux, Solaris, FreeBSD, and other POSIX-compliant systems
+**MITRE ATT&CK:** T1071.001 (Web Protocols), T1573.002 (Asymmetric Cryptography)
+
+**What it is:** A POSIX-compatible implant written in C. Supports automated file exfiltration, configurable beacon intervals, and TLS-encrypted communications with the C2 server.
+
+**Operational significance:** Aeris is the CIA's general-purpose implant for Unix-like systems. Where Athena targets Windows, Aeris provides equivalent capability across the POSIX ecosystem. Its portability across Linux, Solaris, and BSD makes it suitable for targeting servers, networking equipment running Unix-like OS, and embedded systems.
+
+## 2.5 AfterMidnight (Windows)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows
+**MITRE ATT&CK:** T1574.001 (DLL Hijacking), T1129 (Shared Modules)
+
+**What it is:** A modular Windows implant framework. The base implant masquerades as a Windows DLL, self-persists, and supports dynamically loaded payload modules called "Gremlins."
+
+**Architecture:** AfterMidnight is a two-component system:
+1. **The platform** — A persistent DLL that loads at boot and provides a framework for loading, executing, and unloading Gremlins
+2. **Gremlins** — Payload modules that provide specific capabilities (keylogging, screen capture, file exfiltration, etc.)
+
+**Operational significance:** The modular design means the platform is deployed once. Capabilities are added and removed as needed without re-deploying the implant. AlphaGremlin is the task scheduling component that manages when and how Gremlins execute.
+
+---
+
+# Chapter 3: Persistence Frameworks
+
+## 3.1 Angelfire / Wolfcreek (Windows)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows
+**MITRE ATT&CK:** T1542.003 (Bootkit), T1014 (Rootkit), T1564.005 (Hidden File System)
+
+**What it is:** A five-component Windows persistence framework that modifies the boot process to load implants before the OS fully initializes.
+
+**The five components:**
+
+1. **Solartime** — Modifies the partition boot sector to load Wolfcreek during the Windows boot process. This executes before the OS kernel is fully loaded, giving the implant a privileged position.
+
+2. **Wolfcreek** — A kernel-mode driver loaded by Solartime. Provides a platform for loading additional user-mode implants. Operates at the kernel level, making it invisible to user-space security tools.
+
+3. **Keystone** — The user-mode component loaded by Wolfcreek. Provides the interface between the kernel-mode persistence mechanism and the operational payloads.
+
+4. **BadMFS** — A covert file system. Creates a hidden storage partition that is invisible to the operating system. Used to store implant components, configuration data, and exfiltrated material where forensic tools cannot easily find them.
+
+5. **Windows Transitory File System** — A component for loading file-based implants without leaving artifacts on the standard file system.
+
+**Operational significance:** Angelfire represents the most sophisticated persistence mechanism in the Vault 7 collection. By modifying the boot sector and loading before the OS, it bypasses all OS-level security controls. The covert filesystem (BadMFS) ensures that even if the OS is forensically imaged, the implant storage is not visible through standard analysis.
+
+**Detection indicators:**
+- Modified partition boot sector (compare against known-good boot sector hashes)
+- Unexpected kernel drivers loaded early in the boot sequence
+- Disk sectors containing data that don't correspond to any visible filesystem
+- Anomalous disk I/O to sectors not mapped to any partition
+
+---
+
+# Chapter 4: Network Device Exploitation
+
+## 4.1 CherryBlossom (Wireless Routers)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA + SRI International (contractor)
+**Platforms:** Wireless routers and access points (multiple vendors)
+**MITRE ATT&CK:** T1557 (Adversary-in-the-Middle), T1200 (Hardware Additions)
+
+**What it is:** A framework for exploiting wireless networking devices (routers, access points). CherryBlossom replaces the router's firmware with an implanted version that provides persistent access to all network traffic flowing through the device.
+
+**Architecture:**
+
+- **Flytrap** — The implanted firmware installed on the target router. Replaces the stock firmware while maintaining normal router functionality. Beacons to the CherryTree server.
+- **CherryTree** — The command-and-control server that manages Flytrap implants. Receives beacons, issues tasking, and collects intercepted data.
+- **CherryWeb** — A browser-based interface for operators to manage implanted devices through the CherryTree server.
+- **Mission** — The tasking system that defines what data to collect from the target network.
+
+**Test infrastructure (from leaked documents):**
+The CherryBlossom test environment used IPs in the 10.6.6.0/24 and 10.6.7.0/24 ranges with VPN tunnels between components. Components included: Mobile VPN server, Flytrap (implanted router), CherryTree server, Beacon, Mission, Snoball, and ICON.
+
+**Operational significance:** Compromising a router gives access to ALL traffic on that network without touching any endpoint. This is a network-level implant that is invisible to endpoint security tools. The target organization's computers, phones, and IoT devices are all monitored through their own infrastructure.
+
+---
+
+# Chapter 5: Air-Gap Operations
+
+## 5.1 Brutal Kangaroo / Drifting Deadline
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Version:** 1.2 (February 2016)
+**Platforms:** Windows, USB removable media
+**MITRE ATT&CK:** T1091 (Replication Through Removable Media), T1052.001 (Exfiltration Over USB)
+
+**What it is:** A toolsuite for penetrating air-gapped networks using infected USB drives as the bridge. Drifting Deadline is the primary component that creates a covert data exfiltration channel across the air gap.
+
+**How it works:** The attack chain:
+1. Infect a USB drive that will be used by someone with access to the air-gapped network
+2. When the USB is inserted into a computer on the air-gapped network, the implant executes
+3. The implant surveys the air-gapped network and collects target data
+4. Collected data is staged on the USB drive in a covert partition
+5. When the USB is returned to an internet-connected computer, the data is exfiltrated
+
+**Operational significance:** Air-gapped networks are considered the gold standard of network security — physically disconnected from the internet. Brutal Kangaroo demonstrates that air gaps can be bridged through the human element: people who move USB drives between connected and disconnected networks.
+
+## 5.2 EzCheese (Removable Media Exploitation)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows
+**MITRE ATT&CK:** T1091 (Replication Through Removable Media), T1203 (Exploitation for Client Execution)
+
+**What it is:** An exploitation tool that triggers when Windows Explorer displays the folder contents of an infected removable drive. The exploit fires automatically when the user browses to the drive — no file execution required.
+
+**Operational significance:** EzCheese lowers the bar for USB-based infection. The target doesn't need to run a file — merely opening the drive in Explorer is sufficient. This makes USB-based attacks more reliable and harder to defend against through user education ("don't run unknown executables" doesn't help when browsing a folder is enough).
+
+---
+
+# Chapter 6: Credential Theft
+
+## 6.1 BothanSpy (Windows SSH)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows (targets Xshell SSH client)
+**MITRE ATT&CK:** T1056.001 (Keylogging), T1555 (Credentials from Password Stores)
+
+**What it is:** A credential harvesting tool that targets the Xshell SSH client on Windows. BothanSpy steals usernames, passwords, and SSH session content from active sessions.
+
+**Operational significance:** SSH credentials provide access to remote servers, often in privileged contexts. A single compromised SSH session can provide access to entire server farms, especially if key-based authentication is used with shared keys.
+
+## 6.2 Gyrfalcon (Linux SSH)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Linux (targets OpenSSH client)
+**MITRE ATT&CK:** T1056.001 (Keylogging), T1555 (Credentials from Password Stores)
+
+**What it is:** The Linux counterpart to BothanSpy. Gyrfalcon captures usernames, passwords, and full session content from OpenSSH client sessions on Linux.
+
+**Together:** BothanSpy (Windows) + Gyrfalcon (Linux) give the CIA coverage of SSH credential theft across both major desktop platforms.
+
+---
+
+# Chapter 7: Collection Tools
+
+## 7.1 ExpressLane (Biometric Collection)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA OTS (Office of Technical Service)
+**Platforms:** Windows (biometric collection systems)
+**MITRE ATT&CK:** T1005 (Data from Local System), T1036 (Masquerading)
+
+**What it is:** A covert biometric data collection tool disguised as a software update for OEM liaison biometric systems. When the CIA provides biometric collection equipment to liaison services (foreign intelligence partners), ExpressLane is installed alongside it. During "software updates," ExpressLane exfiltrates the biometric data the liaison service has collected.
+
+**Operational significance:** This reveals that the CIA backdoors equipment it provides to allied intelligence services. The "updates" are a pretext for data theft from partners. This has significant implications for international intelligence cooperation trust.
+
+## 7.2 CouchPotato (Video Capture)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows
+**MITRE ATT&CK:** T1125 (Video Capture), T1557 (Adversary-in-the-Middle)
+
+**What it is:** A remote tool for capturing RTSP/H.264 video streams from IP cameras and surveillance systems.
+
+## 7.3 ELSA (WiFi Geolocation)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows
+**MITRE ATT&CK:** T1614 (System Location Discovery), T1016 (System Network Configuration Discovery)
+
+**What it is:** A WiFi geolocation tool that collects nearby WiFi access point MAC addresses and signal strengths, then uses geo-lookup databases to determine the target device's physical location. No GPS required — WiFi access point mapping provides location data accurate to tens of meters in urban areas.
+
+---
+
+# Chapter 8: Support & Evasion Tools
+
+## 8.1 Dumbo (Device Disabling)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Windows
+**MITRE ATT&CK:** T1562.001 (Disable or Modify Tools)
+
+**What it is:** A tool that identifies and disables webcams and microphones on a target system. Designed to support physical access operations — when CIA operatives need to enter a room without being recorded, Dumbo is deployed first to disable all recording devices.
+
+**Operational significance:** Dumbo bridges cyber and physical operations. It is deployed before a physical entry to ensure the operative is not captured on webcam or recorded by microphone. This implies a workflow where cyber access precedes physical access.
+
+## 8.2 HighRise (Android SMS Proxy)
+
+**Classification:** SECRET//NOFORN
+**Developer:** CIA EDG
+**Platforms:** Android
+**MITRE ATT&CK:** T1437 (Application Layer Protocol), T1573 (Encrypted Channel)
+
+**What it is:** An Android application that redirects SMS messages through an internet-based relay to a CIA listening post. Used for covert communications — a compromised Android device can serve as an SMS relay without the user's knowledge.
+
+## 8.3 Protego (Missile Control)
+
+**Classification:** SECRET//NOFORN
+**Platforms:** Embedded PIC microcontroller
+
+**What it is:** PIC-based missile control system electronics with GPS targeting, in-flight guidance, and self-destruct capability. This is not a cyber tool — it is a weapons system. Its presence in the Vault 7 archive indicates CIA EDG's scope extends beyond cyber operations into weapons engineering.
+
+---
+
+# Chapter 9: False Flag Operations (Umbrage)
+
+## 9.1 The Umbrage Program
+
+**Organizational home:** Remote Development Branch (RDB), CIA
+**Purpose:** Borrow techniques, code, and TTPs from other threat actors to create false attribution in CIA cyber operations.
+
+**Components:**
+- **Hacking Team Source Dump Map** — Analysis and cataloging of Hacking Team's source code (leaked in 2015) for reusable components
+- **PIQUE Assessments** — Evaluations of tools and techniques from other actors for potential adoption
+- **Component Library** — Reusable code modules borrowed from other actors' tools
+
+**Operational significance:** Umbrage allows the CIA to conduct operations that, if discovered, would be attributed to other threat actors rather than the CIA. By using code patterns, techniques, and infrastructure associated with Russian, Chinese, or criminal groups, the CIA can create false flag cyber operations.
+
+**Implications for threat intelligence:** If the CIA maintains a library of other actors' techniques specifically for false flag operations, then code-level attribution of cyber attacks is fundamentally unreliable. The presence of "Russian" or "Chinese" code in an attack does not prove the attack was conducted by Russia or China — it may prove only that the attacker had access to that code.
+
+---
+
+# Chapter 10: Operational Patterns & Detection
+
+## 10.1 Common CIA Patterns
+
+Analyzing the Vault 7 tools reveals patterns that can inform both red team operations and defensive detection:
+
+**Persistence preferences:**
+- DLL hijacking (Athena, AfterMidnight) — low visibility, survives reboots
+- Kernel modules (OutlawCountry, Wolfcreek) — highest privilege, hardest to detect
+- Boot sector modification (Solartime/Angelfire) — pre-OS execution
+- Service manipulation (Athena) — using legitimate Windows services as hosts
+
+**Communication patterns:**
+- Beacon-based C2 (periodic check-in rather than persistent connection)
+- TLS encryption for C2 traffic
+- Blending with legitimate traffic (HTTP/HTTPS beaconing)
+- Use of legitimate process (srvhost) as network client
+
+**Deployment patterns:**
+- Physical access tools (Dumbo, ExpressLane) suggest human deployment alongside technical
+- USB-based tools (Brutal Kangaroo, EzCheese) for air-gapped targets
+- Router firmware replacement (CherryBlossom) for network-level access
+
+## 10.2 Detection Framework
+
+**Endpoint indicators:**
+- Unexpected DLLs with export forwarding chains
+- Modified boot sectors
+- Kernel modules not associated with installed software
+- Minifilter drivers with no corresponding application
+- Services enabled that should be disabled (RemoteAccess)
+- Modified srvhost service groupings
+
+**Network indicators:**
+- Periodic beacon traffic to unknown external IPs from system processes
+- Router firmware hash mismatches against known-good images
+- DNS/HTTP traffic taking unexpected routes
+- TLS connections from processes that shouldn't be making network calls
+
+**Physical indicators:**
+- Webcam/microphone drivers disabled without user action
+- USB devices with hidden partitions
+- Biometric system behavior during "software updates"
+
+---
+
+*This manual is a living document. As additional Vault 7 PDFs are parsed, deeper technical details
+will be extracted and incorporated. Regenerate with `python mosaic.py manual`.*

storage/db.py

@@ -4,7 +4,7 @@ import logging
 import re
 from pathlib import Path
 from typing import Optional
-from datetime import datetime
+import datetime
 
 from .models import (
     Document, ExtractedTool, TTP, Tradecraft, Entity,
@@ -172,7 +172,7 @@ class MosaicDB:
     def update_document_extraction_time(self, doc_id: str):
         self.conn.execute(
             "UPDATE documents SET last_extracted_at = ? WHERE id = ?",
-            (datetime.utcnow().isoformat(), doc_id)
+            (datetime.datetime.datetime.now(datetime.timezone.utc).isoformat(), doc_id)
         )
         self.conn.commit()
 
@@ -289,8 +289,8 @@ class MosaicDB:
             ON CONFLICT(doc_id, extractor)
             DO UPDATE SET status = ?, error = ?, last_run = ?, token_count = ?
         """, (
-            doc_id, extractor, status, error, datetime.utcnow().isoformat(), token_count,
+            doc_id, extractor, status, error, datetime.datetime.now(datetime.timezone.utc).isoformat(), token_count,
-            status, error, datetime.utcnow().isoformat(), token_count
+            status, error, datetime.datetime.now(datetime.timezone.utc).isoformat(), token_count
         ))
         self.conn.commit()
 

storage/jsonl_store.py

@@ -3,7 +3,7 @@ import json
 import hashlib
 import logging
 from pathlib import Path
-from datetime import datetime
+import datetime
 from typing import Optional
 
 from .db import MosaicDB
@@ -45,7 +45,7 @@ class JSONLStore:
                 logger.info("No data for category: %s", cat)
                 continue
 
-            timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+            timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
             filename = f"{cat}_{timestamp}.jsonl"
             filepath = self.output_dir / filename
 

tests/test_mosaic.py

@@ -304,7 +304,7 @@ class TestTextUtils:
 
     def test_clean_text(self):
         from utils.text_utils import clean_text
-        assert clean_text("hello\x00world") == "hello world"  # null bytes removed (control chars)
+        assert clean_text("hello\x00world") == "helloworld"  # null bytes stripped
         assert clean_text("line1\r\nline2") == "line1\nline2"
         assert clean_text("a\n\n\n\nb") == "a\n\nb"