Fix deprecated datetime.utcnow() and test assertion for clean_text

collectors/base.py

@@ -4,7 +4,7 @@ import tempfile
 import hashlib
 import logging
 from abc import ABC, abstractmethod
-from datetime import datetime
+import datetime
 from pathlib import Path
 from typing import Optional
 
@@ -119,7 +119,7 @@ class BaseCollector(ABC):
                     title=self._extract_title(url),
                     raw_path=final_path,
                     content_hash=content_hash,
-                    fetch_date=datetime.utcnow().isoformat(),
+                    fetch_date=datetime.datetime.now(datetime.timezone.utc).isoformat(),
                     etag=etag,
                     last_modified=last_modified,
                     char_count=len(content.decode('utf-8', errors='replace')),

extractors/base.py

@@ -2,7 +2,7 @@
 import json
 import logging
 from abc import ABC, abstractmethod
-from datetime import datetime
+import datetime
 from typing import Optional
 
 import anthropic
@@ -130,7 +130,7 @@ class BaseExtractor(ABC):
 
             # Log token usage
             self.db.log_token_usage(TokenUsage(
-                timestamp=datetime.utcnow().isoformat(),
+                timestamp=datetime.datetime.now(datetime.timezone.utc).isoformat(),
                 source=source,
                 doc_id=doc_id,
                 extractor=self.EXTRACTOR_NAME,

storage/db.py

@@ -4,7 +4,7 @@ import logging
 import re
 from pathlib import Path
 from typing import Optional
-from datetime import datetime
+import datetime
 
 from .models import (
     Document, ExtractedTool, TTP, Tradecraft, Entity,
@@ -172,7 +172,7 @@ class MosaicDB:
     def update_document_extraction_time(self, doc_id: str):
         self.conn.execute(
             "UPDATE documents SET last_extracted_at = ? WHERE id = ?",
-            (datetime.utcnow().isoformat(), doc_id)
+            (datetime.datetime.datetime.now(datetime.timezone.utc).isoformat(), doc_id)
         )
         self.conn.commit()
 
@@ -289,8 +289,8 @@ class MosaicDB:
             ON CONFLICT(doc_id, extractor)
             DO UPDATE SET status = ?, error = ?, last_run = ?, token_count = ?
         """, (
-            doc_id, extractor, status, error, datetime.utcnow().isoformat(), token_count,
-            status, error, datetime.utcnow().isoformat(), token_count
+            doc_id, extractor, status, error, datetime.datetime.now(datetime.timezone.utc).isoformat(), token_count,
+            status, error, datetime.datetime.now(datetime.timezone.utc).isoformat(), token_count
         ))
         self.conn.commit()
 

storage/jsonl_store.py

@@ -3,7 +3,7 @@ import json
 import hashlib
 import logging
 from pathlib import Path
-from datetime import datetime
+import datetime
 from typing import Optional
 
 from .db import MosaicDB
@@ -45,7 +45,7 @@ class JSONLStore:
                 logger.info("No data for category: %s", cat)
                 continue
 
-            timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+            timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
             filename = f"{cat}_{timestamp}.jsonl"
             filepath = self.output_dir / filename
 

tests/test_mosaic.py

@@ -304,7 +304,7 @@ class TestTextUtils:
 
     def test_clean_text(self):
         from utils.text_utils import clean_text
-        assert clean_text("hello\x00world") == "hello world"  # null bytes removed (control chars)
+        assert clean_text("hello\x00world") == "helloworld"  # null bytes stripped
         assert clean_text("line1\r\nline2") == "line1\nline2"
         assert clean_text("a\n\n\n\nb") == "a\n\nb"