Add extractors: TTP, tool, tradecraft, surveillance, infrastructure, entity, MITRE mapper + prompts

extractors/entity_extractor.py

@@ -0,0 +1,42 @@
+"""Entity extractor (people, orgs, programs, codenames, units)."""
+import logging
+
+from .base import BaseExtractor
+from storage.models import Entity
+from analysis.prompts import ENTITY_SYSTEM_PROMPT, ENTITY_EXTRACTION_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+class EntityExtractor(BaseExtractor):
+    EXTRACTOR_NAME = "entity"
+
+    def get_system_prompt(self) -> str:
+        return ENTITY_SYSTEM_PROMPT
+
+    def get_extraction_prompt(self, text: str) -> str:
+        return ENTITY_EXTRACTION_PROMPT.format(document=text)
+
+    def get_json_schema(self) -> dict:
+        return {"properties": {"items": {"type": "array"}}, "required": ["items"]}
+
+    def process_response(self, data: dict, doc_id: str) -> int:
+        items = data.get('items', [])
+        if isinstance(data, list):
+            items = data
+        count = 0
+        for item in items:
+            try:
+                entity = Entity(
+                    name=item.get('name', ''),
+                    type=item.get('type', ''),
+                    description=item.get('description', ''),
+                    relationships=item.get('relationships', []),
+                    source_doc_ids=[doc_id],
+                    source_context=item.get('source_context', '')[:500],
+                )
+                self.db.insert_entity(entity)
+                count += 1
+            except Exception as e:
+                logger.warning("Failed to process entity item: %s", e)
+        return count