feat: RTL_433 protocol database import - iteration 1/5
- Expanded protocol database from 18 → 299 signatures (16.6x increase) - Imported 281 protocols from RTL_433 open-source database (286 total devices) - Created automated import script: scripts/import_rtl433_protocols.py - Generated rtl433_protocols_imported.py with timing/frequency/modulation data - Updated protocol_database.py to include RTL433_PROTOCOLS - All 26 tests passing Breakdown by category: - Weather: 116 protocols - Sensors: 36 protocols - TPMS: 25 protocols - Security: 23 protocols - Home Automation: 18 protocols - Other: 50+ protocols Frequency coverage: - 433.92 MHz: 248 protocols - 315.00 MHz: 32 protocols - 915.00 MHz: 1 protocol This provides comprehensive coverage of Sub-GHz IoT devices for accurate identification from raw RF captures.
This commit is contained in:
leetcrypt
+1
-1
@@ -4,6 +4,6 @@ GigLez API Package
|
||||
FastAPI application with environment-aware configuration
|
||||
"""
|
||||
|
||||
from .main import app
|
||||
from .main_simple import app
|
||||
|
||||
__all__ = ['app']
|
||||
|
||||
@@ -24,7 +24,7 @@ from src.parser.gps_extractor import GPSFilenameExtractor
|
||||
from src.matcher.strategies import (
|
||||
ExactMatcher,
|
||||
FrequencyMatcher,
|
||||
BitPatternMatcher,
|
||||
PatternMatcher,
|
||||
TimingMatcher,
|
||||
RTL433DecoderStrategy,
|
||||
PatternBasedStrategy
|
||||
@@ -81,7 +81,7 @@ def get_matcher_engine():
|
||||
# Add all strategies in order of confidence
|
||||
_matcher_engine.add_strategy(ExactMatcher()) # Exact protocol + frequency
|
||||
_matcher_engine.add_strategy(FrequencyMatcher()) # Frequency-based
|
||||
_matcher_engine.add_strategy(BitPatternMatcher()) # Bit pattern matching
|
||||
_matcher_engine.add_strategy(PatternMatcher()) # Bit pattern matching
|
||||
_matcher_engine.add_strategy(TimingMatcher()) # Timing-based
|
||||
_matcher_engine.add_strategy(RTL433DecoderStrategy()) # RTL_433 decoder
|
||||
_matcher_engine.add_strategy(PatternBasedStrategy()) # Pattern decoder (NEW!)
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
"""
|
||||
Enhanced Capture Upload Endpoints
|
||||
|
||||
Features:
|
||||
- Automatic device matching on upload
|
||||
- Manual device labeling support
|
||||
- Photo upload for unknown devices
|
||||
- Training data collection
|
||||
"""
|
||||
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status, BackgroundTasks
|
||||
from sqlalchemy.orm import Session
|
||||
from loguru import logger
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from src.api.dependencies import get_db, optional_auth, get_storage
|
||||
from src.database.models import (
|
||||
User, Capture, Session as CaptureSession,
|
||||
CaptureMatch, Device, ManualIdentification
|
||||
)
|
||||
from src.parser.sub_parser import parse_sub_file
|
||||
from src.gps.validator import validate_gps_coordinates
|
||||
from src.matcher.engine import SignatureMatcher
|
||||
from config.settings import settings
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ENHANCED UPLOAD ENDPOINT
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/upload/enhanced")
|
||||
async def upload_captures_enhanced(
|
||||
manifest: str = Form(..., description="JSON manifest with GPS data and manual labels"),
|
||||
files: List[UploadFile] = File(..., description=".sub files to upload"),
|
||||
background_tasks: BackgroundTasks = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth),
|
||||
storage = Depends(get_storage)
|
||||
):
|
||||
"""
|
||||
Enhanced upload with automatic matching and manual labeling
|
||||
|
||||
**Manifest Format**:
|
||||
```json
|
||||
{
|
||||
"session_uuid": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"captures": [
|
||||
{
|
||||
"filename": "capture_001.sub",
|
||||
"latitude": 40.7128,
|
||||
"longitude": -74.0060,
|
||||
"accuracy": 5.0,
|
||||
"altitude": 10.5,
|
||||
"timestamp": "2026-01-12T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"manual_labels": [
|
||||
{
|
||||
"filename": "capture_001.sub",
|
||||
"device_id": 123,
|
||||
"source": "manual_existing"
|
||||
},
|
||||
{
|
||||
"filename": "capture_002.sub",
|
||||
"device_type": "Garage Door Opener",
|
||||
"manufacturer": "Chamberlain",
|
||||
"model": "891LM",
|
||||
"notes": "Captured from my garage",
|
||||
"source": "manual_new"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Parse and validate manifest
|
||||
2. For each file:
|
||||
- Deduplicate by hash
|
||||
- Validate GPS
|
||||
- Parse .sub file
|
||||
- Store file
|
||||
- **Run automatic matching (NEW)**
|
||||
- **Apply manual labels if provided (NEW)**
|
||||
- Save capture with matches
|
||||
3. Return results with device identifications
|
||||
"""
|
||||
|
||||
try:
|
||||
# Parse manifest
|
||||
manifest_data = json.loads(manifest)
|
||||
session_uuid = manifest_data.get("session_uuid")
|
||||
captures_manifest = manifest_data.get("captures", [])
|
||||
manual_labels_list = manifest_data.get("manual_labels", [])
|
||||
|
||||
logger.info(f"Enhanced upload: {len(files)} files, {len(manual_labels_list)} manual labels")
|
||||
|
||||
# Create manual labels lookup
|
||||
manual_labels_map = {label['filename']: label for label in manual_labels_list}
|
||||
|
||||
# Get or create session
|
||||
capture_session = None
|
||||
if session_uuid:
|
||||
capture_session = db.query(CaptureSession).filter_by(
|
||||
session_uuid=session_uuid
|
||||
).first()
|
||||
|
||||
if not capture_session:
|
||||
capture_session = CaptureSession(
|
||||
session_uuid=session_uuid,
|
||||
user_id=user.id if user else None,
|
||||
started_at=datetime.utcnow()
|
||||
)
|
||||
db.add(capture_session)
|
||||
db.flush()
|
||||
|
||||
# Create filename to manifest mapping
|
||||
manifest_map = {c['filename']: c for c in captures_manifest}
|
||||
|
||||
# Initialize matcher
|
||||
matcher = SignatureMatcher(db)
|
||||
|
||||
# Process files
|
||||
results = {"successful": [], "failed": []}
|
||||
|
||||
for file in files:
|
||||
try:
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
|
||||
# Compute SHA256 hash
|
||||
file_hash = hashlib.sha256(content).hexdigest()
|
||||
|
||||
# Check if already exists
|
||||
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
|
||||
if existing:
|
||||
results["successful"].append({
|
||||
"filename": file.filename,
|
||||
"status": "duplicate",
|
||||
"file_hash": file_hash,
|
||||
"matches": [] # Could retrieve existing matches
|
||||
})
|
||||
continue
|
||||
|
||||
# Get manifest entry
|
||||
manifest_entry = manifest_map.get(file.filename)
|
||||
if not manifest_entry:
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": "Missing manifest entry"
|
||||
})
|
||||
continue
|
||||
|
||||
# Validate GPS
|
||||
latitude = float(manifest_entry['latitude'])
|
||||
longitude = float(manifest_entry['longitude'])
|
||||
accuracy = manifest_entry.get('accuracy')
|
||||
|
||||
if not validate_gps_coordinates(latitude, longitude, accuracy):
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Invalid GPS coordinates: ({latitude}, {longitude})"
|
||||
})
|
||||
continue
|
||||
|
||||
# Parse .sub file
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix='.sub') as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
metadata = parse_sub_file(tmp_path)
|
||||
except Exception as e:
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Failed to parse .sub file: {e}"
|
||||
})
|
||||
continue
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
# Store file
|
||||
storage_path = storage.save_file(
|
||||
file_hash=file_hash,
|
||||
content=content,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
# Create capture record
|
||||
capture = Capture(
|
||||
file_hash=file_hash,
|
||||
session_id=capture_session.id if capture_session else None,
|
||||
user_id=user.id if user else None,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
altitude=manifest_entry.get('altitude'),
|
||||
gps_accuracy=accuracy,
|
||||
captured_at=datetime.fromisoformat(
|
||||
manifest_entry['timestamp'].replace('Z', '+00:00')
|
||||
),
|
||||
frequency=metadata.frequency,
|
||||
modulation=metadata.modulation,
|
||||
preset=metadata.preset,
|
||||
protocol=metadata.protocol,
|
||||
bit_length=metadata.bit_length,
|
||||
key_data=metadata.key_data,
|
||||
timing_element=metadata.timing_element,
|
||||
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
|
||||
raw_format=metadata.file_format,
|
||||
file_path=storage_path,
|
||||
file_size=len(content)
|
||||
)
|
||||
|
||||
db.add(capture)
|
||||
db.flush() # Get capture ID
|
||||
|
||||
# =====================================================================
|
||||
# NEW: Automatic Device Matching
|
||||
# =====================================================================
|
||||
matches = []
|
||||
try:
|
||||
# Run matching engine
|
||||
match_results = matcher.match(metadata)
|
||||
|
||||
# Store top 10 matches
|
||||
for match_result in match_results[:10]:
|
||||
device = db.query(Device).filter_by(id=match_result.device_id).first()
|
||||
if not device:
|
||||
continue
|
||||
|
||||
# Create CaptureMatch record
|
||||
capture_match = CaptureMatch(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
confidence=match_result.confidence,
|
||||
match_method=match_result.method,
|
||||
match_details=match_result.details or {}
|
||||
)
|
||||
db.add(capture_match)
|
||||
|
||||
matches.append({
|
||||
"device_id": device.id,
|
||||
"manufacturer": device.manufacturer,
|
||||
"model": device.model,
|
||||
"device_type": device.device_type,
|
||||
"confidence": match_result.confidence,
|
||||
"method": match_result.method
|
||||
})
|
||||
|
||||
# Set best match as capture's device_id
|
||||
if matches:
|
||||
capture.device_id = matches[0]["device_id"]
|
||||
capture.match_confidence = matches[0]["confidence"]
|
||||
capture.match_method = matches[0]["method"]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Matching failed for {file.filename}: {e}")
|
||||
# Continue without matches
|
||||
|
||||
# =====================================================================
|
||||
# NEW: Manual Device Labeling
|
||||
# =====================================================================
|
||||
manual_label = manual_labels_map.get(file.filename)
|
||||
if manual_label:
|
||||
await handle_manual_label(
|
||||
db=db,
|
||||
capture=capture,
|
||||
manual_label=manual_label,
|
||||
user=user
|
||||
)
|
||||
|
||||
# Commit this capture
|
||||
db.commit()
|
||||
|
||||
results["successful"].append({
|
||||
"filename": file.filename,
|
||||
"status": "uploaded",
|
||||
"file_hash": file_hash,
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol,
|
||||
"matches": matches,
|
||||
"manual_label": manual_label is not None
|
||||
})
|
||||
|
||||
logger.debug(f"Capture uploaded with {len(matches)} matches: {file.filename}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process file {file.filename}: {e}")
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": str(e)
|
||||
})
|
||||
|
||||
logger.success(f"Enhanced upload complete: {len(results['successful'])} successful, {len(results['failed'])} failed")
|
||||
|
||||
return results
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON manifest: {e}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Enhanced upload failed: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Upload failed: {e}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MANUAL LABEL HANDLING
|
||||
# =============================================================================
|
||||
|
||||
async def handle_manual_label(
|
||||
db: Session,
|
||||
capture: Capture,
|
||||
manual_label: dict,
|
||||
user: Optional[User]
|
||||
):
|
||||
"""
|
||||
Process manual device label from user
|
||||
|
||||
Two types:
|
||||
1. manual_existing: User selected from existing device database
|
||||
2. manual_new: User created new device entry
|
||||
"""
|
||||
|
||||
source = manual_label.get("source", "manual")
|
||||
|
||||
if source == "manual_existing":
|
||||
# User selected existing device
|
||||
device_id = manual_label.get("device_id")
|
||||
if not device_id:
|
||||
return
|
||||
|
||||
device = db.query(Device).filter_by(id=device_id).first()
|
||||
if not device:
|
||||
logger.warning(f"Manual label references non-existent device: {device_id}")
|
||||
return
|
||||
|
||||
# Override automatic match with manual selection
|
||||
capture.device_id = device_id
|
||||
capture.match_confidence = 1.0 # User-confirmed
|
||||
capture.match_method = "manual_selection"
|
||||
|
||||
# Create ManualIdentification record
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device_id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False, # Needs community verification
|
||||
verification_count=1 # Initial submission counts as 1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
logger.info(f"Manual label applied: capture {capture.id} -> device {device_id}")
|
||||
|
||||
elif source == "manual_new":
|
||||
# User created new device
|
||||
device_type = manual_label.get("device_type")
|
||||
manufacturer = manual_label.get("manufacturer")
|
||||
model = manual_label.get("model")
|
||||
|
||||
if not all([device_type, manufacturer, model]):
|
||||
logger.warning("Incomplete manual_new label data")
|
||||
return
|
||||
|
||||
# Check if device already exists
|
||||
existing_device = db.query(Device).filter_by(
|
||||
manufacturer=manufacturer,
|
||||
model=model
|
||||
).first()
|
||||
|
||||
if existing_device:
|
||||
device = existing_device
|
||||
else:
|
||||
# Create new device
|
||||
device = Device(
|
||||
manufacturer=manufacturer,
|
||||
model=model,
|
||||
device_type=device_type,
|
||||
typical_frequency=capture.frequency,
|
||||
protocol=capture.protocol,
|
||||
source="community",
|
||||
verified=False, # Needs verification
|
||||
metadata_json={"submitter_user_id": user.id if user else None}
|
||||
)
|
||||
db.add(device)
|
||||
db.flush() # Get device ID
|
||||
|
||||
# Link capture to device
|
||||
capture.device_id = device.id
|
||||
capture.match_confidence = 1.0
|
||||
capture.match_method = "manual_new_device"
|
||||
|
||||
# Create ManualIdentification record
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False,
|
||||
verification_count=1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
logger.info(f"New device created from manual label: {manufacturer} {model}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BATCH MATCHING ENDPOINT (For existing captures)
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/match_batch")
|
||||
async def match_captures_batch(
|
||||
limit: int = 100,
|
||||
background_tasks: BackgroundTasks = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth)
|
||||
):
|
||||
"""
|
||||
Run matching engine on existing captures that don't have device identifications
|
||||
|
||||
Use this to backfill matches for captures uploaded before automatic matching was implemented.
|
||||
"""
|
||||
|
||||
# Find captures without device_id
|
||||
unmatched_captures = db.query(Capture).filter(
|
||||
Capture.device_id.is_(None)
|
||||
).limit(limit).all()
|
||||
|
||||
if not unmatched_captures:
|
||||
return {
|
||||
"message": "No unmatched captures found",
|
||||
"processed": 0
|
||||
}
|
||||
|
||||
logger.info(f"Batch matching {len(unmatched_captures)} captures")
|
||||
|
||||
matcher = SignatureMatcher(db)
|
||||
matched_count = 0
|
||||
|
||||
for capture in unmatched_captures:
|
||||
try:
|
||||
# Reconstruct SignalMetadata from capture
|
||||
from src.parser.metadata import SignalMetadata
|
||||
|
||||
metadata = SignalMetadata()
|
||||
metadata.frequency = capture.frequency
|
||||
metadata.protocol = capture.protocol
|
||||
metadata.modulation = capture.modulation
|
||||
metadata.bit_length = capture.bit_length
|
||||
metadata.key_data = capture.key_data
|
||||
metadata.timing_element = capture.timing_element
|
||||
|
||||
if capture.raw_data:
|
||||
# Parse raw_data string back to list
|
||||
metadata.raw_data = eval(capture.raw_data) if capture.raw_data.startswith('[') else None
|
||||
|
||||
# Run matching
|
||||
match_results = matcher.match(metadata)
|
||||
|
||||
if match_results:
|
||||
# Store top match
|
||||
best_match = match_results[0]
|
||||
capture.device_id = best_match.device_id
|
||||
capture.match_confidence = best_match.confidence
|
||||
capture.match_method = best_match.method
|
||||
|
||||
# Store all matches in CaptureMatch table
|
||||
for match_result in match_results[:10]:
|
||||
capture_match = CaptureMatch(
|
||||
capture_id=capture.id,
|
||||
device_id=match_result.device_id,
|
||||
confidence=match_result.confidence,
|
||||
match_method=match_result.method,
|
||||
match_details=match_result.details or {}
|
||||
)
|
||||
db.add(capture_match)
|
||||
|
||||
matched_count += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Batch matching failed for capture {capture.id}: {e}")
|
||||
continue
|
||||
|
||||
db.commit()
|
||||
|
||||
logger.success(f"Batch matching complete: {matched_count}/{len(unmatched_captures)} matched")
|
||||
|
||||
return {
|
||||
"message": f"Processed {len(unmatched_captures)} captures",
|
||||
"matched": matched_count,
|
||||
"unmatched": len(unmatched_captures) - matched_count
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
"""
|
||||
Enhanced Capture Upload Endpoint with Robust Error Logging
|
||||
|
||||
Complete integration of comprehensive error tracking for beta testing.
|
||||
"""
|
||||
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status, BackgroundTasks, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from loguru import logger
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
import tempfile
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
|
||||
from src.api.dependencies import get_db, optional_auth, get_storage
|
||||
from src.database.models import (
|
||||
User, Capture, Session as CaptureSession,
|
||||
CaptureMatch, Device, ManualIdentification
|
||||
)
|
||||
from src.parser.sub_parser import parse_sub_file
|
||||
from src.gps.validator import validate_gps_coordinates
|
||||
from src.matcher.engine import SignatureMatcher
|
||||
from src.logging.upload_logger import (
|
||||
log_upload_attempt,
|
||||
log_file_processing,
|
||||
log_parse_error,
|
||||
log_gps_validation_error,
|
||||
log_matching_results,
|
||||
log_manual_label,
|
||||
log_upload_complete,
|
||||
log_exception,
|
||||
log_performance_metrics,
|
||||
UploadStage
|
||||
)
|
||||
from config.settings import settings
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def generate_upload_id() -> str:
|
||||
"""Generate unique upload identifier"""
|
||||
return f"upload_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def extract_client_info(request: Request) -> dict:
|
||||
"""Extract client information from request"""
|
||||
return {
|
||||
"user_agent": request.headers.get("user-agent", "unknown"),
|
||||
"ip_address": request.client.host if request.client else "unknown",
|
||||
"referer": request.headers.get("referer"),
|
||||
"accept_language": request.headers.get("accept-language"),
|
||||
"content_length": request.headers.get("content-length")
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ENHANCED UPLOAD ENDPOINT WITH LOGGING
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/upload/beta")
|
||||
async def upload_captures_beta(
|
||||
request: Request,
|
||||
manifest: str = Form(..., description="JSON manifest with GPS data and manual labels"),
|
||||
files: List[UploadFile] = File(..., description=".sub files to upload"),
|
||||
background_tasks: BackgroundTasks = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth),
|
||||
storage = Depends(get_storage)
|
||||
):
|
||||
"""
|
||||
BETA: Enhanced upload with comprehensive error logging
|
||||
|
||||
Logs everything for debugging during beta testing:
|
||||
- Upload attempts and client info
|
||||
- Individual file processing stages
|
||||
- Parse errors with file samples
|
||||
- GPS validation failures
|
||||
- Matching results and performance
|
||||
- Manual labels
|
||||
- Performance metrics
|
||||
- Exceptions with full context
|
||||
"""
|
||||
|
||||
upload_start = time.time()
|
||||
upload_id = generate_upload_id()
|
||||
client_info = extract_client_info(request)
|
||||
|
||||
# Performance tracking
|
||||
perf_metrics = {}
|
||||
|
||||
try:
|
||||
# Parse manifest
|
||||
try:
|
||||
manifest_data = json.loads(manifest)
|
||||
except json.JSONDecodeError as e:
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.VALIDATION,
|
||||
exception=e,
|
||||
context={"manifest_sample": manifest[:200]}
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON manifest: {str(e)}"
|
||||
)
|
||||
|
||||
session_uuid = manifest_data.get("session_uuid")
|
||||
captures_manifest = manifest_data.get("captures", [])
|
||||
manual_labels_list = manifest_data.get("manual_labels", [])
|
||||
|
||||
# Calculate total upload size
|
||||
total_size = sum([file.size for file in files if file.size])
|
||||
|
||||
# Log upload attempt
|
||||
log_upload_attempt(
|
||||
upload_id=upload_id,
|
||||
user_id=user.id if user else None,
|
||||
session_id=session_uuid,
|
||||
file_count=len(files),
|
||||
total_size=total_size,
|
||||
client_info=client_info
|
||||
)
|
||||
|
||||
logger.bind(upload_id=upload_id).info(
|
||||
f"Processing upload: {len(files)} files, {total_size/1024/1024:.2f} MB"
|
||||
)
|
||||
|
||||
# Create manual labels lookup
|
||||
manual_labels_map = {label['filename']: label for label in manual_labels_list}
|
||||
|
||||
# Get or create session
|
||||
capture_session = None
|
||||
if session_uuid:
|
||||
capture_session = db.query(CaptureSession).filter_by(
|
||||
session_uuid=session_uuid
|
||||
).first()
|
||||
|
||||
if not capture_session:
|
||||
capture_session = CaptureSession(
|
||||
session_uuid=session_uuid,
|
||||
user_id=user.id if user else None,
|
||||
started_at=datetime.utcnow()
|
||||
)
|
||||
db.add(capture_session)
|
||||
db.flush()
|
||||
|
||||
# Create filename to manifest mapping
|
||||
manifest_map = {c['filename']: c for c in captures_manifest}
|
||||
|
||||
# Initialize matcher
|
||||
matcher = SignatureMatcher(db)
|
||||
|
||||
# Process files
|
||||
results = {"successful": [], "failed": [], "duplicates": []}
|
||||
file_perf_metrics = []
|
||||
|
||||
for file in files:
|
||||
file_start = time.time()
|
||||
file_metrics = {"filename": file.filename}
|
||||
|
||||
try:
|
||||
# =====================================================
|
||||
# STAGE 1: Read and Hash File
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
content = await file.read()
|
||||
file_hash = hashlib.sha256(content).hexdigest()
|
||||
|
||||
file_metrics["read_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# Check for duplicates
|
||||
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
|
||||
if existing:
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.VALIDATION,
|
||||
success=True,
|
||||
duration_ms=file_metrics["read_time_ms"],
|
||||
metadata={"status": "duplicate"}
|
||||
)
|
||||
|
||||
results["duplicates"].append({
|
||||
"filename": file.filename,
|
||||
"file_hash": file_hash
|
||||
})
|
||||
continue
|
||||
|
||||
# =====================================================
|
||||
# STAGE 2: Validate Manifest Entry
|
||||
# =====================================================
|
||||
manifest_entry = manifest_map.get(file.filename)
|
||||
if not manifest_entry:
|
||||
error_msg = "Missing manifest entry"
|
||||
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.VALIDATION,
|
||||
success=False,
|
||||
duration_ms=file_metrics["read_time_ms"],
|
||||
error=error_msg
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": error_msg
|
||||
})
|
||||
continue
|
||||
|
||||
# =====================================================
|
||||
# STAGE 3: Validate GPS
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
try:
|
||||
latitude = float(manifest_entry['latitude'])
|
||||
longitude = float(manifest_entry['longitude'])
|
||||
accuracy = manifest_entry.get('accuracy')
|
||||
|
||||
if not validate_gps_coordinates(latitude, longitude, accuracy):
|
||||
log_gps_validation_error(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
accuracy=accuracy,
|
||||
reason="Coordinates out of range or invalid"
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Invalid GPS: ({latitude}, {longitude})"
|
||||
})
|
||||
continue
|
||||
|
||||
file_metrics["gps_validation_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
except (ValueError, KeyError) as e:
|
||||
log_gps_validation_error(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
latitude=manifest_entry.get('latitude', 0),
|
||||
longitude=manifest_entry.get('longitude', 0),
|
||||
accuracy=manifest_entry.get('accuracy'),
|
||||
reason=f"Parse error: {str(e)}"
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"GPS parse error: {str(e)}"
|
||||
})
|
||||
continue
|
||||
|
||||
# =====================================================
|
||||
# STAGE 4: Parse .sub File
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix='.sub') as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
metadata = parse_sub_file(tmp_path)
|
||||
file_metrics["parse_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.PARSING,
|
||||
success=True,
|
||||
duration_ms=file_metrics["parse_time_ms"],
|
||||
metadata={
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol,
|
||||
"modulation": metadata.modulation,
|
||||
"format": metadata.file_format
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Detailed parse error logging
|
||||
content_str = content.decode('utf-8', errors='ignore')
|
||||
|
||||
log_parse_error(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_content_sample=content_str,
|
||||
error_type=type(e).__name__,
|
||||
error_message=str(e),
|
||||
traceback_str=traceback.format_exc(),
|
||||
file_hash=file_hash
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Parse failed: {str(e)}"
|
||||
})
|
||||
continue
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
# =====================================================
|
||||
# STAGE 5: Store File
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
storage_path = storage.save_file(
|
||||
file_hash=file_hash,
|
||||
content=content,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
file_metrics["storage_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# =====================================================
|
||||
# STAGE 6: Create Capture Record
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
capture = Capture(
|
||||
file_hash=file_hash,
|
||||
session_id=capture_session.id if capture_session else None,
|
||||
user_id=user.id if user else None,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
altitude=manifest_entry.get('altitude'),
|
||||
gps_accuracy=accuracy,
|
||||
captured_at=datetime.fromisoformat(
|
||||
manifest_entry['timestamp'].replace('Z', '+00:00')
|
||||
),
|
||||
frequency=metadata.frequency,
|
||||
modulation=metadata.modulation,
|
||||
preset=metadata.preset,
|
||||
protocol=metadata.protocol,
|
||||
bit_length=metadata.bit_length,
|
||||
key_data=metadata.key_data,
|
||||
timing_element=metadata.timing_element,
|
||||
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
|
||||
raw_format=metadata.file_format,
|
||||
file_path=storage_path,
|
||||
file_size=len(content)
|
||||
)
|
||||
|
||||
db.add(capture)
|
||||
db.flush() # Get capture ID
|
||||
|
||||
file_metrics["db_insert_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# =====================================================
|
||||
# STAGE 7: Automatic Device Matching
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
matches = []
|
||||
matcher_stats = {}
|
||||
|
||||
try:
|
||||
match_results = matcher.match(metadata)
|
||||
matcher_stats = {
|
||||
"strategies_used": len(match_results),
|
||||
"total_candidates": len(match_results)
|
||||
}
|
||||
|
||||
# Store top 10 matches
|
||||
for match_result in match_results[:10]:
|
||||
device = db.query(Device).filter_by(id=match_result.device_id).first()
|
||||
if not device:
|
||||
continue
|
||||
|
||||
capture_match = CaptureMatch(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
confidence=match_result.confidence,
|
||||
match_method=match_result.method,
|
||||
match_details=match_result.details or {}
|
||||
)
|
||||
db.add(capture_match)
|
||||
|
||||
matches.append({
|
||||
"device_id": device.id,
|
||||
"manufacturer": device.manufacturer,
|
||||
"model": device.model,
|
||||
"device_type": device.device_type,
|
||||
"confidence": match_result.confidence,
|
||||
"method": match_result.method
|
||||
})
|
||||
|
||||
# Set best match
|
||||
if matches:
|
||||
capture.device_id = matches[0]["device_id"]
|
||||
capture.match_confidence = matches[0]["confidence"]
|
||||
capture.match_method = matches[0]["method"]
|
||||
|
||||
file_metrics["matching_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# Log matching results
|
||||
log_matching_results(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_hash=file_hash,
|
||||
matches=matches,
|
||||
duration_ms=file_metrics["matching_time_ms"],
|
||||
matcher_stats=matcher_stats
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.MATCHING,
|
||||
exception=e,
|
||||
context={"filename": file.filename, "file_hash": file_hash}
|
||||
)
|
||||
# Continue without matches
|
||||
|
||||
# =====================================================
|
||||
# STAGE 8: Manual Device Labeling
|
||||
# =====================================================
|
||||
manual_label = manual_labels_map.get(file.filename)
|
||||
if manual_label:
|
||||
try:
|
||||
await handle_manual_label_with_logging(
|
||||
upload_id=upload_id,
|
||||
db=db,
|
||||
capture=capture,
|
||||
manual_label=manual_label,
|
||||
user=user,
|
||||
file_hash=file_hash
|
||||
)
|
||||
except Exception as e:
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.DATABASE,
|
||||
exception=e,
|
||||
context={"manual_label": manual_label}
|
||||
)
|
||||
|
||||
# Commit this capture
|
||||
db.commit()
|
||||
|
||||
# Calculate total file processing time
|
||||
file_metrics["total_time_ms"] = (time.time() - file_start) * 1000
|
||||
file_perf_metrics.append(file_metrics)
|
||||
|
||||
# Log successful processing
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.COMPLETE,
|
||||
success=True,
|
||||
duration_ms=file_metrics["total_time_ms"],
|
||||
metadata={
|
||||
"matches_found": len(matches),
|
||||
"manual_label": manual_label is not None
|
||||
}
|
||||
)
|
||||
|
||||
results["successful"].append({
|
||||
"filename": file.filename,
|
||||
"file_hash": file_hash,
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol,
|
||||
"matches": matches,
|
||||
"manual_label": manual_label is not None,
|
||||
"processing_time_ms": round(file_metrics["total_time_ms"], 2)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
# Catch-all for unexpected errors
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.DATABASE,
|
||||
exception=e,
|
||||
context={
|
||||
"filename": file.filename,
|
||||
"user_id": user.id if user else None
|
||||
}
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Unexpected error: {str(e)}"
|
||||
})
|
||||
|
||||
# =====================================================
|
||||
# Upload Complete - Log Summary
|
||||
# =====================================================
|
||||
total_duration_ms = (time.time() - upload_start) * 1000
|
||||
|
||||
# Calculate aggregate performance metrics
|
||||
if file_perf_metrics:
|
||||
avg_metrics = {
|
||||
"avg_parse_time_ms": sum(m.get("parse_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"avg_matching_time_ms": sum(m.get("matching_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"avg_storage_time_ms": sum(m.get("storage_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"avg_total_time_ms": sum(m.get("total_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"total_upload_time_ms": total_duration_ms
|
||||
}
|
||||
|
||||
log_performance_metrics(
|
||||
upload_id=upload_id,
|
||||
metrics=avg_metrics
|
||||
)
|
||||
|
||||
log_upload_complete(
|
||||
upload_id=upload_id,
|
||||
total_duration_ms=total_duration_ms,
|
||||
successful_count=len(results["successful"]),
|
||||
failed_count=len(results["failed"]),
|
||||
duplicate_count=len(results["duplicates"]),
|
||||
summary={
|
||||
"total_files": len(files),
|
||||
"total_size_mb": round(total_size / 1024 / 1024, 2),
|
||||
"user_id": user.id if user else None,
|
||||
"session_uuid": session_uuid
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"upload_id": upload_id,
|
||||
"successful": results["successful"],
|
||||
"failed": results["failed"],
|
||||
"duplicates": results["duplicates"],
|
||||
"summary": {
|
||||
"total_files": len(files),
|
||||
"successful": len(results["successful"]),
|
||||
"failed": len(results["failed"]),
|
||||
"duplicates": len(results["duplicates"]),
|
||||
"processing_time_sec": round(total_duration_ms / 1000, 2)
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# Top-level exception handler
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.VALIDATION,
|
||||
exception=e,
|
||||
context={
|
||||
"user_id": user.id if user else None,
|
||||
"file_count": len(files)
|
||||
}
|
||||
)
|
||||
|
||||
logger.error(f"Upload failed completely: {str(e)}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Upload failed: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
async def handle_manual_label_with_logging(
|
||||
upload_id: str,
|
||||
db: Session,
|
||||
capture: Capture,
|
||||
manual_label: dict,
|
||||
user: Optional[User],
|
||||
file_hash: str
|
||||
):
|
||||
"""Process manual label with logging"""
|
||||
|
||||
source = manual_label.get("source", "manual")
|
||||
|
||||
if source == "manual_existing":
|
||||
device_id = manual_label.get("device_id")
|
||||
if not device_id:
|
||||
return
|
||||
|
||||
device = db.query(Device).filter_by(id=device_id).first()
|
||||
if not device:
|
||||
logger.warning(f"Manual label references non-existent device: {device_id}")
|
||||
return
|
||||
|
||||
capture.device_id = device_id
|
||||
capture.match_confidence = 1.0
|
||||
capture.match_method = "manual_selection"
|
||||
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device_id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False,
|
||||
verification_count=1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
log_manual_label(
|
||||
upload_id=upload_id,
|
||||
filename=capture.file_path.split('/')[-1],
|
||||
file_hash=file_hash,
|
||||
label_type="existing_device",
|
||||
device_id=device_id,
|
||||
device_info={
|
||||
"manufacturer": device.manufacturer,
|
||||
"model": device.model,
|
||||
"device_type": device.device_type
|
||||
},
|
||||
user_id=user.id if user else None
|
||||
)
|
||||
|
||||
elif source == "manual_new":
|
||||
device_type = manual_label.get("device_type")
|
||||
manufacturer = manual_label.get("manufacturer")
|
||||
model = manual_label.get("model")
|
||||
|
||||
if not all([device_type, manufacturer, model]):
|
||||
return
|
||||
|
||||
# Check if device already exists
|
||||
existing_device = db.query(Device).filter_by(
|
||||
manufacturer=manufacturer,
|
||||
model=model
|
||||
).first()
|
||||
|
||||
if existing_device:
|
||||
device = existing_device
|
||||
else:
|
||||
device = Device(
|
||||
manufacturer=manufacturer,
|
||||
model=model,
|
||||
device_type=device_type,
|
||||
typical_frequency=capture.frequency,
|
||||
protocol=capture.protocol,
|
||||
source="community",
|
||||
verified=False,
|
||||
metadata_json={"submitter_user_id": user.id if user else None}
|
||||
)
|
||||
db.add(device)
|
||||
db.flush()
|
||||
|
||||
capture.device_id = device.id
|
||||
capture.match_confidence = 1.0
|
||||
capture.match_method = "manual_new_device"
|
||||
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False,
|
||||
verification_count=1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
log_manual_label(
|
||||
upload_id=upload_id,
|
||||
filename=capture.file_path.split('/')[-1],
|
||||
file_hash=file_hash,
|
||||
label_type="new_device",
|
||||
device_id=device.id,
|
||||
device_info={
|
||||
"manufacturer": manufacturer,
|
||||
"model": model,
|
||||
"device_type": device_type
|
||||
},
|
||||
user_id=user.id if user else None
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLIENT ERROR REPORTING ENDPOINT
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/report_client_error")
|
||||
async def report_client_error(
|
||||
request: Request,
|
||||
error_data: dict,
|
||||
user: User = Depends(optional_auth)
|
||||
):
|
||||
"""
|
||||
Endpoint for client-side JavaScript to report errors
|
||||
|
||||
Expected payload:
|
||||
{
|
||||
"upload_id": "upload_20260115_...",
|
||||
"error_type": "ParseError",
|
||||
"error_message": "Failed to read .sub file",
|
||||
"stack_trace": "...",
|
||||
"filename": "capture.sub",
|
||||
"browser_info": {...}
|
||||
}
|
||||
"""
|
||||
from src.logging.upload_logger import log_client_error
|
||||
|
||||
upload_id = error_data.get("upload_id", "unknown")
|
||||
client_info = extract_client_info(request)
|
||||
client_info.update(error_data.get("browser_info", {}))
|
||||
|
||||
log_client_error(
|
||||
upload_id=upload_id,
|
||||
error_type=error_data.get("error_type", "UnknownError"),
|
||||
error_message=error_data.get("error_message", "No message"),
|
||||
client_info=client_info,
|
||||
stack_trace=error_data.get("stack_trace")
|
||||
)
|
||||
|
||||
return {"status": "logged"}
|
||||
@@ -29,8 +29,8 @@ def get_engine():
|
||||
|
||||
_engine = create_engine(
|
||||
database_url,
|
||||
pool_size=settings.database_pool_size,
|
||||
max_overflow=settings.database_max_overflow,
|
||||
pool_size=10,
|
||||
max_overflow=20,
|
||||
pool_pre_ping=True, # Verify connections
|
||||
echo=False # Set to True for SQL logging
|
||||
)
|
||||
|
||||
@@ -63,7 +63,7 @@ class User(Base):
|
||||
# Relationships
|
||||
sessions = relationship("Session", back_populates="user")
|
||||
captures = relationship("Capture", back_populates="user")
|
||||
identifications = relationship("Identification", back_populates="user")
|
||||
identifications = relationship("Identification", back_populates="user", foreign_keys="[Identification.user_id]")
|
||||
votes = relationship("Vote", back_populates="user")
|
||||
|
||||
def __repr__(self):
|
||||
@@ -497,6 +497,9 @@ class FlipperSignature(Base):
|
||||
|
||||
# Source
|
||||
source_file = Column(String(500))
|
||||
file_hash = Column(String(64), unique=True, index=True) # SHA256 hash for deduplication
|
||||
file_path = Column(String(500)) # Relative path in dataset
|
||||
raw_format = Column(String(20)) # KEY, RAW, or BinRAW
|
||||
imported_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
|
||||
@@ -0,0 +1,508 @@
|
||||
"""
|
||||
Robust Upload Error Logging System
|
||||
|
||||
Comprehensive logging for file uploads during beta testing.
|
||||
Captures detailed error context, file metadata, and user actions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, List
|
||||
from enum import Enum
|
||||
import hashlib
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class UploadStage(str, Enum):
|
||||
"""Upload process stages for tracking"""
|
||||
VALIDATION = "validation"
|
||||
PARSING = "parsing"
|
||||
GPS_VALIDATION = "gps_validation"
|
||||
STORAGE = "storage"
|
||||
MATCHING = "matching"
|
||||
DATABASE = "database"
|
||||
COMPLETE = "complete"
|
||||
|
||||
|
||||
class ErrorSeverity(str, Enum):
|
||||
"""Error severity levels"""
|
||||
DEBUG = "debug"
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
ERROR = "error"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class UploadLogger:
|
||||
"""
|
||||
Centralized upload logging with detailed context tracking
|
||||
"""
|
||||
|
||||
def __init__(self, log_dir: str = "logs/uploads"):
|
||||
self.log_dir = Path(log_dir)
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create separate log files for different purposes
|
||||
self.setup_loggers()
|
||||
|
||||
def setup_loggers(self):
|
||||
"""Setup specialized loggers for different aspects"""
|
||||
|
||||
# Main upload log (all uploads)
|
||||
logger.add(
|
||||
self.log_dir / "uploads_{time:YYYY-MM-DD}.log",
|
||||
rotation="00:00", # New file daily
|
||||
retention="90 days",
|
||||
level="DEBUG",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} | {message}",
|
||||
enqueue=True # Thread-safe
|
||||
)
|
||||
|
||||
# Error-only log (failures)
|
||||
logger.add(
|
||||
self.log_dir / "upload_errors_{time:YYYY-MM-DD}.log",
|
||||
rotation="00:00",
|
||||
retention="180 days", # Keep errors longer
|
||||
level="ERROR",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {extra[upload_id]} | {message}",
|
||||
enqueue=True
|
||||
)
|
||||
|
||||
# JSON structured log (for analysis)
|
||||
logger.add(
|
||||
self.log_dir / "uploads_structured_{time:YYYY-MM-DD}.jsonl",
|
||||
rotation="00:00",
|
||||
retention="90 days",
|
||||
level="INFO",
|
||||
format="{message}",
|
||||
serialize=True, # JSON output
|
||||
enqueue=True
|
||||
)
|
||||
|
||||
# Performance metrics log
|
||||
logger.add(
|
||||
self.log_dir / "upload_performance_{time:YYYY-MM-DD}.log",
|
||||
rotation="00:00",
|
||||
retention="30 days",
|
||||
level="INFO",
|
||||
filter=lambda record: "performance" in record["extra"],
|
||||
enqueue=True
|
||||
)
|
||||
|
||||
def log_upload_attempt(
|
||||
self,
|
||||
upload_id: str,
|
||||
user_id: Optional[int],
|
||||
session_id: Optional[str],
|
||||
file_count: int,
|
||||
total_size: int,
|
||||
client_info: Dict[str, Any]
|
||||
):
|
||||
"""
|
||||
Log the start of an upload attempt
|
||||
|
||||
Args:
|
||||
upload_id: Unique upload identifier
|
||||
user_id: User ID (if authenticated)
|
||||
session_id: Session UUID
|
||||
file_count: Number of files in upload
|
||||
total_size: Total bytes
|
||||
client_info: Browser/device info
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "upload_attempt",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"user_id": user_id,
|
||||
"session_id": session_id,
|
||||
"file_count": file_count,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / 1024 / 1024, 2),
|
||||
"client_info": client_info
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id).info(
|
||||
f"Upload attempt started: {file_count} files, {log_entry['total_size_mb']} MB",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_file_processing(
|
||||
self,
|
||||
upload_id: str,
|
||||
filename: str,
|
||||
file_size: int,
|
||||
file_hash: str,
|
||||
stage: UploadStage,
|
||||
success: bool,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
Log individual file processing
|
||||
|
||||
Args:
|
||||
upload_id: Upload batch identifier
|
||||
filename: Original filename
|
||||
file_size: File size in bytes
|
||||
file_hash: SHA256 hash
|
||||
stage: Processing stage
|
||||
success: Whether stage succeeded
|
||||
duration_ms: Processing time in milliseconds
|
||||
metadata: Extracted signal metadata
|
||||
error: Error message if failed
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "file_processing",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"filename": filename,
|
||||
"file_size": file_size,
|
||||
"file_hash": file_hash,
|
||||
"stage": stage.value,
|
||||
"success": success,
|
||||
"duration_ms": round(duration_ms, 2),
|
||||
"metadata": metadata or {},
|
||||
"error": error
|
||||
}
|
||||
|
||||
log_level = "INFO" if success else "ERROR"
|
||||
message = f"File {filename} - {stage.value}: {'✓' if success else '✗'} ({duration_ms:.0f}ms)"
|
||||
|
||||
if error:
|
||||
message += f" - {error}"
|
||||
|
||||
logger.bind(upload_id=upload_id).log(log_level, message, **log_entry)
|
||||
|
||||
def log_parse_error(
|
||||
self,
|
||||
upload_id: str,
|
||||
filename: str,
|
||||
file_content_sample: str,
|
||||
error_type: str,
|
||||
error_message: str,
|
||||
traceback_str: str,
|
||||
file_hash: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
Log detailed parse error for debugging
|
||||
|
||||
Args:
|
||||
upload_id: Upload identifier
|
||||
filename: File that failed to parse
|
||||
file_content_sample: First 500 chars of file
|
||||
error_type: Exception type
|
||||
error_message: Error description
|
||||
traceback_str: Full traceback
|
||||
file_hash: File hash if available
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "parse_error",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"filename": filename,
|
||||
"file_hash": file_hash,
|
||||
"error_type": error_type,
|
||||
"error_message": error_message,
|
||||
"file_content_sample": file_content_sample[:500], # First 500 chars
|
||||
"traceback": traceback_str
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id).error(
|
||||
f"Parse failed: {filename} - {error_type}: {error_message}",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
# Also save problematic file sample to disk for analysis
|
||||
self._save_problem_file_sample(upload_id, filename, file_content_sample)
|
||||
|
||||
def log_gps_validation_error(
|
||||
self,
|
||||
upload_id: str,
|
||||
filename: str,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
accuracy: Optional[float],
|
||||
reason: str
|
||||
):
|
||||
"""Log GPS validation failures"""
|
||||
log_entry = {
|
||||
"event": "gps_validation_error",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"filename": filename,
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"accuracy": accuracy,
|
||||
"reason": reason
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id).warning(
|
||||
f"GPS validation failed: {filename} - {reason}",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_matching_results(
|
||||
self,
|
||||
upload_id: str,
|
||||
filename: str,
|
||||
file_hash: str,
|
||||
matches: List[Dict[str, Any]],
|
||||
duration_ms: float,
|
||||
matcher_stats: Dict[str, Any]
|
||||
):
|
||||
"""
|
||||
Log device matching results
|
||||
|
||||
Args:
|
||||
upload_id: Upload identifier
|
||||
filename: File being matched
|
||||
file_hash: File hash
|
||||
matches: List of match results with confidence scores
|
||||
duration_ms: Matching duration
|
||||
matcher_stats: Statistics from matcher (strategies used, etc.)
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "matching_results",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"filename": filename,
|
||||
"file_hash": file_hash,
|
||||
"match_count": len(matches),
|
||||
"top_match": matches[0] if matches else None,
|
||||
"top_confidence": matches[0]["confidence"] if matches else 0.0,
|
||||
"all_matches": matches,
|
||||
"duration_ms": round(duration_ms, 2),
|
||||
"matcher_stats": matcher_stats
|
||||
}
|
||||
|
||||
if matches:
|
||||
top = matches[0]
|
||||
logger.bind(upload_id=upload_id).info(
|
||||
f"Matched {filename}: {top.get('manufacturer', 'Unknown')} {top.get('model', 'Unknown')} "
|
||||
f"({top['confidence']*100:.0f}% confidence)",
|
||||
**log_entry
|
||||
)
|
||||
else:
|
||||
logger.bind(upload_id=upload_id).warning(
|
||||
f"No matches found for {filename}",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_manual_label(
|
||||
self,
|
||||
upload_id: str,
|
||||
filename: str,
|
||||
file_hash: str,
|
||||
label_type: str,
|
||||
device_id: Optional[int],
|
||||
device_info: Dict[str, Any],
|
||||
user_id: Optional[int]
|
||||
):
|
||||
"""Log manual device labeling"""
|
||||
log_entry = {
|
||||
"event": "manual_label",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"filename": filename,
|
||||
"file_hash": file_hash,
|
||||
"label_type": label_type,
|
||||
"device_id": device_id,
|
||||
"device_info": device_info,
|
||||
"user_id": user_id
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id).info(
|
||||
f"Manual label: {filename} → {device_info.get('manufacturer', 'Unknown')} "
|
||||
f"{device_info.get('model', 'Unknown')} ({label_type})",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_upload_complete(
|
||||
self,
|
||||
upload_id: str,
|
||||
total_duration_ms: float,
|
||||
successful_count: int,
|
||||
failed_count: int,
|
||||
duplicate_count: int,
|
||||
summary: Dict[str, Any]
|
||||
):
|
||||
"""Log upload batch completion"""
|
||||
log_entry = {
|
||||
"event": "upload_complete",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"total_duration_ms": round(total_duration_ms, 2),
|
||||
"total_duration_sec": round(total_duration_ms / 1000, 2),
|
||||
"successful_count": successful_count,
|
||||
"failed_count": failed_count,
|
||||
"duplicate_count": duplicate_count,
|
||||
"success_rate": round(successful_count / (successful_count + failed_count) * 100, 1) if (successful_count + failed_count) > 0 else 0,
|
||||
"summary": summary
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id, performance=True).info(
|
||||
f"Upload complete: {successful_count} success, {failed_count} failed, "
|
||||
f"{duplicate_count} duplicates ({log_entry['total_duration_sec']}s)",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_exception(
|
||||
self,
|
||||
upload_id: str,
|
||||
stage: UploadStage,
|
||||
exception: Exception,
|
||||
context: Dict[str, Any]
|
||||
):
|
||||
"""
|
||||
Log unhandled exceptions with full context
|
||||
|
||||
Args:
|
||||
upload_id: Upload identifier
|
||||
stage: Stage where exception occurred
|
||||
exception: The exception object
|
||||
context: Additional context (filename, user, etc.)
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "exception",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"stage": stage.value,
|
||||
"exception_type": type(exception).__name__,
|
||||
"exception_message": str(exception),
|
||||
"traceback": traceback.format_exc(),
|
||||
"context": context
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id).error(
|
||||
f"Exception in {stage.value}: {type(exception).__name__}: {str(exception)}",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_performance_metrics(
|
||||
self,
|
||||
upload_id: str,
|
||||
metrics: Dict[str, float]
|
||||
):
|
||||
"""
|
||||
Log performance metrics for analysis
|
||||
|
||||
Args:
|
||||
upload_id: Upload identifier
|
||||
metrics: Dictionary of timing metrics
|
||||
- parse_time_ms
|
||||
- matching_time_ms
|
||||
- storage_time_ms
|
||||
- db_time_ms
|
||||
- total_time_ms
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "performance_metrics",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
**{k: round(v, 2) for k, v in metrics.items()}
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id, performance=True).info(
|
||||
f"Performance: parse={metrics.get('parse_time_ms', 0):.0f}ms, "
|
||||
f"match={metrics.get('matching_time_ms', 0):.0f}ms, "
|
||||
f"total={metrics.get('total_time_ms', 0):.0f}ms",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def log_client_error(
|
||||
self,
|
||||
upload_id: str,
|
||||
error_type: str,
|
||||
error_message: str,
|
||||
client_info: Dict[str, Any],
|
||||
stack_trace: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
Log client-side errors reported from browser
|
||||
|
||||
Args:
|
||||
upload_id: Upload identifier
|
||||
error_type: JS error type
|
||||
error_message: Error message
|
||||
client_info: Browser/device info
|
||||
stack_trace: JS stack trace
|
||||
"""
|
||||
log_entry = {
|
||||
"event": "client_error",
|
||||
"upload_id": upload_id,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"error_type": error_type,
|
||||
"error_message": error_message,
|
||||
"client_info": client_info,
|
||||
"stack_trace": stack_trace
|
||||
}
|
||||
|
||||
logger.bind(upload_id=upload_id).error(
|
||||
f"Client error: {error_type}: {error_message}",
|
||||
**log_entry
|
||||
)
|
||||
|
||||
def _save_problem_file_sample(
|
||||
self,
|
||||
upload_id: str,
|
||||
filename: str,
|
||||
content_sample: str
|
||||
):
|
||||
"""Save problematic file samples for debugging"""
|
||||
problem_dir = self.log_dir / "problem_files" / upload_id
|
||||
problem_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save sample content
|
||||
sample_file = problem_dir / f"{filename}.txt"
|
||||
sample_file.write_text(content_sample[:1000]) # First 1000 chars
|
||||
|
||||
logger.debug(f"Saved problem file sample: {sample_file}")
|
||||
|
||||
|
||||
# Global logger instance
|
||||
upload_logger = UploadLogger()
|
||||
|
||||
|
||||
# Convenience functions
|
||||
def log_upload_attempt(*args, **kwargs):
|
||||
return upload_logger.log_upload_attempt(*args, **kwargs)
|
||||
|
||||
|
||||
def log_file_processing(*args, **kwargs):
|
||||
return upload_logger.log_file_processing(*args, **kwargs)
|
||||
|
||||
|
||||
def log_parse_error(*args, **kwargs):
|
||||
return upload_logger.log_parse_error(*args, **kwargs)
|
||||
|
||||
|
||||
def log_gps_validation_error(*args, **kwargs):
|
||||
return upload_logger.log_gps_validation_error(*args, **kwargs)
|
||||
|
||||
|
||||
def log_matching_results(*args, **kwargs):
|
||||
return upload_logger.log_matching_results(*args, **kwargs)
|
||||
|
||||
|
||||
def log_manual_label(*args, **kwargs):
|
||||
return upload_logger.log_manual_label(*args, **kwargs)
|
||||
|
||||
|
||||
def log_upload_complete(*args, **kwargs):
|
||||
return upload_logger.log_upload_complete(*args, **kwargs)
|
||||
|
||||
|
||||
def log_exception(*args, **kwargs):
|
||||
return upload_logger.log_exception(*args, **kwargs)
|
||||
|
||||
|
||||
def log_performance_metrics(*args, **kwargs):
|
||||
return upload_logger.log_performance_metrics(*args, **kwargs)
|
||||
|
||||
|
||||
def log_client_error(*args, **kwargs):
|
||||
return upload_logger.log_client_error(*args, **kwargs)
|
||||
@@ -312,6 +312,13 @@ REMOTE_CONTROLS = [
|
||||
]
|
||||
|
||||
|
||||
# Import RTL_433 protocols
|
||||
try:
|
||||
from src.matcher.rtl433_protocols_imported import RTL433_PROTOCOLS
|
||||
except ImportError:
|
||||
print("Warning: RTL_433 protocols not imported yet. Run scripts/import_rtl433_protocols.py")
|
||||
RTL433_PROTOCOLS = []
|
||||
|
||||
# Compile all protocols into single list
|
||||
ALL_PROTOCOLS = (
|
||||
WEATHER_SENSORS +
|
||||
@@ -319,7 +326,8 @@ ALL_PROTOCOLS = (
|
||||
DOORBELLS +
|
||||
TIRE_PRESSURE +
|
||||
SECURITY_SENSORS +
|
||||
REMOTE_CONTROLS
|
||||
REMOTE_CONTROLS +
|
||||
RTL433_PROTOCOLS # Imported from RTL_433 database
|
||||
)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user