Add extractors: TTP, tool, tradecraft, surveillance, infrastructure, entity, MITRE mapper + prompts

extractors/surveillance.py

@@ -0,0 +1,45 @@
+"""Surveillance program/capability extractor for Mosaic."""
+import logging
+
+from .base import BaseExtractor
+from storage.models import TTP
+from analysis.prompts import SURVEILLANCE_SYSTEM_PROMPT, SURVEILLANCE_EXTRACTION_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+class SurveillanceExtractor(BaseExtractor):
+    EXTRACTOR_NAME = "surveillance"
+
+    def get_system_prompt(self) -> str:
+        return SURVEILLANCE_SYSTEM_PROMPT
+
+    def get_extraction_prompt(self, text: str) -> str:
+        return SURVEILLANCE_EXTRACTION_PROMPT.format(document=text)
+
+    def get_json_schema(self) -> dict:
+        return {"properties": {"items": {"type": "array"}}, "required": ["items"]}
+
+    def process_response(self, data: dict, doc_id: str) -> int:
+        items = data.get('items', [])
+        if isinstance(data, list):
+            items = data
+        count = 0
+        for item in items:
+            try:
+                # Store surveillance as TTPs with surveillance category
+                ttp = TTP(
+                    technique=item.get('program', item.get('technique', '')),
+                    category='surveillance',
+                    description=item.get('description', ''),
+                    actor=item.get('operator', item.get('actor', None)),
+                    mitre_id=item.get('mitre_id'),
+                    source_doc_ids=[doc_id],
+                    source_context=item.get('source_context', '')[:500],
+                )
+                ttp.compute_dedup_key()
+                if self.db.insert_ttp(ttp):
+                    count += 1
+            except Exception as e:
+                logger.warning("Failed to process surveillance item: %s", e)
+        return count