feat: RTL_433 protocol database import - iteration 1/5
- Expanded protocol database from 18 → 299 signatures (16.6x increase) - Imported 281 protocols from RTL_433 open-source database (286 total devices) - Created automated import script: scripts/import_rtl433_protocols.py - Generated rtl433_protocols_imported.py with timing/frequency/modulation data - Updated protocol_database.py to include RTL433_PROTOCOLS - All 26 tests passing Breakdown by category: - Weather: 116 protocols - Sensors: 36 protocols - TPMS: 25 protocols - Security: 23 protocols - Home Automation: 18 protocols - Other: 50+ protocols Frequency coverage: - 433.92 MHz: 248 protocols - 315.00 MHz: 32 protocols - 915.00 MHz: 1 protocol This provides comprehensive coverage of Sub-GHz IoT devices for accurate identification from raw RF captures.
This commit is contained in:
leetcrypt
+1
-1
@@ -4,6 +4,6 @@ GigLez API Package
|
||||
FastAPI application with environment-aware configuration
|
||||
"""
|
||||
|
||||
from .main import app
|
||||
from .main_simple import app
|
||||
|
||||
__all__ = ['app']
|
||||
|
||||
@@ -24,7 +24,7 @@ from src.parser.gps_extractor import GPSFilenameExtractor
|
||||
from src.matcher.strategies import (
|
||||
ExactMatcher,
|
||||
FrequencyMatcher,
|
||||
BitPatternMatcher,
|
||||
PatternMatcher,
|
||||
TimingMatcher,
|
||||
RTL433DecoderStrategy,
|
||||
PatternBasedStrategy
|
||||
@@ -81,7 +81,7 @@ def get_matcher_engine():
|
||||
# Add all strategies in order of confidence
|
||||
_matcher_engine.add_strategy(ExactMatcher()) # Exact protocol + frequency
|
||||
_matcher_engine.add_strategy(FrequencyMatcher()) # Frequency-based
|
||||
_matcher_engine.add_strategy(BitPatternMatcher()) # Bit pattern matching
|
||||
_matcher_engine.add_strategy(PatternMatcher()) # Bit pattern matching
|
||||
_matcher_engine.add_strategy(TimingMatcher()) # Timing-based
|
||||
_matcher_engine.add_strategy(RTL433DecoderStrategy()) # RTL_433 decoder
|
||||
_matcher_engine.add_strategy(PatternBasedStrategy()) # Pattern decoder (NEW!)
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
"""
|
||||
Enhanced Capture Upload Endpoints
|
||||
|
||||
Features:
|
||||
- Automatic device matching on upload
|
||||
- Manual device labeling support
|
||||
- Photo upload for unknown devices
|
||||
- Training data collection
|
||||
"""
|
||||
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status, BackgroundTasks
|
||||
from sqlalchemy.orm import Session
|
||||
from loguru import logger
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from src.api.dependencies import get_db, optional_auth, get_storage
|
||||
from src.database.models import (
|
||||
User, Capture, Session as CaptureSession,
|
||||
CaptureMatch, Device, ManualIdentification
|
||||
)
|
||||
from src.parser.sub_parser import parse_sub_file
|
||||
from src.gps.validator import validate_gps_coordinates
|
||||
from src.matcher.engine import SignatureMatcher
|
||||
from config.settings import settings
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ENHANCED UPLOAD ENDPOINT
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/upload/enhanced")
|
||||
async def upload_captures_enhanced(
|
||||
manifest: str = Form(..., description="JSON manifest with GPS data and manual labels"),
|
||||
files: List[UploadFile] = File(..., description=".sub files to upload"),
|
||||
background_tasks: BackgroundTasks = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth),
|
||||
storage = Depends(get_storage)
|
||||
):
|
||||
"""
|
||||
Enhanced upload with automatic matching and manual labeling
|
||||
|
||||
**Manifest Format**:
|
||||
```json
|
||||
{
|
||||
"session_uuid": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"captures": [
|
||||
{
|
||||
"filename": "capture_001.sub",
|
||||
"latitude": 40.7128,
|
||||
"longitude": -74.0060,
|
||||
"accuracy": 5.0,
|
||||
"altitude": 10.5,
|
||||
"timestamp": "2026-01-12T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"manual_labels": [
|
||||
{
|
||||
"filename": "capture_001.sub",
|
||||
"device_id": 123,
|
||||
"source": "manual_existing"
|
||||
},
|
||||
{
|
||||
"filename": "capture_002.sub",
|
||||
"device_type": "Garage Door Opener",
|
||||
"manufacturer": "Chamberlain",
|
||||
"model": "891LM",
|
||||
"notes": "Captured from my garage",
|
||||
"source": "manual_new"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Parse and validate manifest
|
||||
2. For each file:
|
||||
- Deduplicate by hash
|
||||
- Validate GPS
|
||||
- Parse .sub file
|
||||
- Store file
|
||||
- **Run automatic matching (NEW)**
|
||||
- **Apply manual labels if provided (NEW)**
|
||||
- Save capture with matches
|
||||
3. Return results with device identifications
|
||||
"""
|
||||
|
||||
try:
|
||||
# Parse manifest
|
||||
manifest_data = json.loads(manifest)
|
||||
session_uuid = manifest_data.get("session_uuid")
|
||||
captures_manifest = manifest_data.get("captures", [])
|
||||
manual_labels_list = manifest_data.get("manual_labels", [])
|
||||
|
||||
logger.info(f"Enhanced upload: {len(files)} files, {len(manual_labels_list)} manual labels")
|
||||
|
||||
# Create manual labels lookup
|
||||
manual_labels_map = {label['filename']: label for label in manual_labels_list}
|
||||
|
||||
# Get or create session
|
||||
capture_session = None
|
||||
if session_uuid:
|
||||
capture_session = db.query(CaptureSession).filter_by(
|
||||
session_uuid=session_uuid
|
||||
).first()
|
||||
|
||||
if not capture_session:
|
||||
capture_session = CaptureSession(
|
||||
session_uuid=session_uuid,
|
||||
user_id=user.id if user else None,
|
||||
started_at=datetime.utcnow()
|
||||
)
|
||||
db.add(capture_session)
|
||||
db.flush()
|
||||
|
||||
# Create filename to manifest mapping
|
||||
manifest_map = {c['filename']: c for c in captures_manifest}
|
||||
|
||||
# Initialize matcher
|
||||
matcher = SignatureMatcher(db)
|
||||
|
||||
# Process files
|
||||
results = {"successful": [], "failed": []}
|
||||
|
||||
for file in files:
|
||||
try:
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
|
||||
# Compute SHA256 hash
|
||||
file_hash = hashlib.sha256(content).hexdigest()
|
||||
|
||||
# Check if already exists
|
||||
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
|
||||
if existing:
|
||||
results["successful"].append({
|
||||
"filename": file.filename,
|
||||
"status": "duplicate",
|
||||
"file_hash": file_hash,
|
||||
"matches": [] # Could retrieve existing matches
|
||||
})
|
||||
continue
|
||||
|
||||
# Get manifest entry
|
||||
manifest_entry = manifest_map.get(file.filename)
|
||||
if not manifest_entry:
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": "Missing manifest entry"
|
||||
})
|
||||
continue
|
||||
|
||||
# Validate GPS
|
||||
latitude = float(manifest_entry['latitude'])
|
||||
longitude = float(manifest_entry['longitude'])
|
||||
accuracy = manifest_entry.get('accuracy')
|
||||
|
||||
if not validate_gps_coordinates(latitude, longitude, accuracy):
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Invalid GPS coordinates: ({latitude}, {longitude})"
|
||||
})
|
||||
continue
|
||||
|
||||
# Parse .sub file
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix='.sub') as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
metadata = parse_sub_file(tmp_path)
|
||||
except Exception as e:
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Failed to parse .sub file: {e}"
|
||||
})
|
||||
continue
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
# Store file
|
||||
storage_path = storage.save_file(
|
||||
file_hash=file_hash,
|
||||
content=content,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
# Create capture record
|
||||
capture = Capture(
|
||||
file_hash=file_hash,
|
||||
session_id=capture_session.id if capture_session else None,
|
||||
user_id=user.id if user else None,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
altitude=manifest_entry.get('altitude'),
|
||||
gps_accuracy=accuracy,
|
||||
captured_at=datetime.fromisoformat(
|
||||
manifest_entry['timestamp'].replace('Z', '+00:00')
|
||||
),
|
||||
frequency=metadata.frequency,
|
||||
modulation=metadata.modulation,
|
||||
preset=metadata.preset,
|
||||
protocol=metadata.protocol,
|
||||
bit_length=metadata.bit_length,
|
||||
key_data=metadata.key_data,
|
||||
timing_element=metadata.timing_element,
|
||||
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
|
||||
raw_format=metadata.file_format,
|
||||
file_path=storage_path,
|
||||
file_size=len(content)
|
||||
)
|
||||
|
||||
db.add(capture)
|
||||
db.flush() # Get capture ID
|
||||
|
||||
# =====================================================================
|
||||
# NEW: Automatic Device Matching
|
||||
# =====================================================================
|
||||
matches = []
|
||||
try:
|
||||
# Run matching engine
|
||||
match_results = matcher.match(metadata)
|
||||
|
||||
# Store top 10 matches
|
||||
for match_result in match_results[:10]:
|
||||
device = db.query(Device).filter_by(id=match_result.device_id).first()
|
||||
if not device:
|
||||
continue
|
||||
|
||||
# Create CaptureMatch record
|
||||
capture_match = CaptureMatch(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
confidence=match_result.confidence,
|
||||
match_method=match_result.method,
|
||||
match_details=match_result.details or {}
|
||||
)
|
||||
db.add(capture_match)
|
||||
|
||||
matches.append({
|
||||
"device_id": device.id,
|
||||
"manufacturer": device.manufacturer,
|
||||
"model": device.model,
|
||||
"device_type": device.device_type,
|
||||
"confidence": match_result.confidence,
|
||||
"method": match_result.method
|
||||
})
|
||||
|
||||
# Set best match as capture's device_id
|
||||
if matches:
|
||||
capture.device_id = matches[0]["device_id"]
|
||||
capture.match_confidence = matches[0]["confidence"]
|
||||
capture.match_method = matches[0]["method"]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Matching failed for {file.filename}: {e}")
|
||||
# Continue without matches
|
||||
|
||||
# =====================================================================
|
||||
# NEW: Manual Device Labeling
|
||||
# =====================================================================
|
||||
manual_label = manual_labels_map.get(file.filename)
|
||||
if manual_label:
|
||||
await handle_manual_label(
|
||||
db=db,
|
||||
capture=capture,
|
||||
manual_label=manual_label,
|
||||
user=user
|
||||
)
|
||||
|
||||
# Commit this capture
|
||||
db.commit()
|
||||
|
||||
results["successful"].append({
|
||||
"filename": file.filename,
|
||||
"status": "uploaded",
|
||||
"file_hash": file_hash,
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol,
|
||||
"matches": matches,
|
||||
"manual_label": manual_label is not None
|
||||
})
|
||||
|
||||
logger.debug(f"Capture uploaded with {len(matches)} matches: {file.filename}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process file {file.filename}: {e}")
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": str(e)
|
||||
})
|
||||
|
||||
logger.success(f"Enhanced upload complete: {len(results['successful'])} successful, {len(results['failed'])} failed")
|
||||
|
||||
return results
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON manifest: {e}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Enhanced upload failed: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Upload failed: {e}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MANUAL LABEL HANDLING
|
||||
# =============================================================================
|
||||
|
||||
async def handle_manual_label(
|
||||
db: Session,
|
||||
capture: Capture,
|
||||
manual_label: dict,
|
||||
user: Optional[User]
|
||||
):
|
||||
"""
|
||||
Process manual device label from user
|
||||
|
||||
Two types:
|
||||
1. manual_existing: User selected from existing device database
|
||||
2. manual_new: User created new device entry
|
||||
"""
|
||||
|
||||
source = manual_label.get("source", "manual")
|
||||
|
||||
if source == "manual_existing":
|
||||
# User selected existing device
|
||||
device_id = manual_label.get("device_id")
|
||||
if not device_id:
|
||||
return
|
||||
|
||||
device = db.query(Device).filter_by(id=device_id).first()
|
||||
if not device:
|
||||
logger.warning(f"Manual label references non-existent device: {device_id}")
|
||||
return
|
||||
|
||||
# Override automatic match with manual selection
|
||||
capture.device_id = device_id
|
||||
capture.match_confidence = 1.0 # User-confirmed
|
||||
capture.match_method = "manual_selection"
|
||||
|
||||
# Create ManualIdentification record
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device_id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False, # Needs community verification
|
||||
verification_count=1 # Initial submission counts as 1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
logger.info(f"Manual label applied: capture {capture.id} -> device {device_id}")
|
||||
|
||||
elif source == "manual_new":
|
||||
# User created new device
|
||||
device_type = manual_label.get("device_type")
|
||||
manufacturer = manual_label.get("manufacturer")
|
||||
model = manual_label.get("model")
|
||||
|
||||
if not all([device_type, manufacturer, model]):
|
||||
logger.warning("Incomplete manual_new label data")
|
||||
return
|
||||
|
||||
# Check if device already exists
|
||||
existing_device = db.query(Device).filter_by(
|
||||
manufacturer=manufacturer,
|
||||
model=model
|
||||
).first()
|
||||
|
||||
if existing_device:
|
||||
device = existing_device
|
||||
else:
|
||||
# Create new device
|
||||
device = Device(
|
||||
manufacturer=manufacturer,
|
||||
model=model,
|
||||
device_type=device_type,
|
||||
typical_frequency=capture.frequency,
|
||||
protocol=capture.protocol,
|
||||
source="community",
|
||||
verified=False, # Needs verification
|
||||
metadata_json={"submitter_user_id": user.id if user else None}
|
||||
)
|
||||
db.add(device)
|
||||
db.flush() # Get device ID
|
||||
|
||||
# Link capture to device
|
||||
capture.device_id = device.id
|
||||
capture.match_confidence = 1.0
|
||||
capture.match_method = "manual_new_device"
|
||||
|
||||
# Create ManualIdentification record
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False,
|
||||
verification_count=1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
logger.info(f"New device created from manual label: {manufacturer} {model}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BATCH MATCHING ENDPOINT (For existing captures)
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/match_batch")
|
||||
async def match_captures_batch(
|
||||
limit: int = 100,
|
||||
background_tasks: BackgroundTasks = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth)
|
||||
):
|
||||
"""
|
||||
Run matching engine on existing captures that don't have device identifications
|
||||
|
||||
Use this to backfill matches for captures uploaded before automatic matching was implemented.
|
||||
"""
|
||||
|
||||
# Find captures without device_id
|
||||
unmatched_captures = db.query(Capture).filter(
|
||||
Capture.device_id.is_(None)
|
||||
).limit(limit).all()
|
||||
|
||||
if not unmatched_captures:
|
||||
return {
|
||||
"message": "No unmatched captures found",
|
||||
"processed": 0
|
||||
}
|
||||
|
||||
logger.info(f"Batch matching {len(unmatched_captures)} captures")
|
||||
|
||||
matcher = SignatureMatcher(db)
|
||||
matched_count = 0
|
||||
|
||||
for capture in unmatched_captures:
|
||||
try:
|
||||
# Reconstruct SignalMetadata from capture
|
||||
from src.parser.metadata import SignalMetadata
|
||||
|
||||
metadata = SignalMetadata()
|
||||
metadata.frequency = capture.frequency
|
||||
metadata.protocol = capture.protocol
|
||||
metadata.modulation = capture.modulation
|
||||
metadata.bit_length = capture.bit_length
|
||||
metadata.key_data = capture.key_data
|
||||
metadata.timing_element = capture.timing_element
|
||||
|
||||
if capture.raw_data:
|
||||
# Parse raw_data string back to list
|
||||
metadata.raw_data = eval(capture.raw_data) if capture.raw_data.startswith('[') else None
|
||||
|
||||
# Run matching
|
||||
match_results = matcher.match(metadata)
|
||||
|
||||
if match_results:
|
||||
# Store top match
|
||||
best_match = match_results[0]
|
||||
capture.device_id = best_match.device_id
|
||||
capture.match_confidence = best_match.confidence
|
||||
capture.match_method = best_match.method
|
||||
|
||||
# Store all matches in CaptureMatch table
|
||||
for match_result in match_results[:10]:
|
||||
capture_match = CaptureMatch(
|
||||
capture_id=capture.id,
|
||||
device_id=match_result.device_id,
|
||||
confidence=match_result.confidence,
|
||||
match_method=match_result.method,
|
||||
match_details=match_result.details or {}
|
||||
)
|
||||
db.add(capture_match)
|
||||
|
||||
matched_count += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Batch matching failed for capture {capture.id}: {e}")
|
||||
continue
|
||||
|
||||
db.commit()
|
||||
|
||||
logger.success(f"Batch matching complete: {matched_count}/{len(unmatched_captures)} matched")
|
||||
|
||||
return {
|
||||
"message": f"Processed {len(unmatched_captures)} captures",
|
||||
"matched": matched_count,
|
||||
"unmatched": len(unmatched_captures) - matched_count
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
"""
|
||||
Enhanced Capture Upload Endpoint with Robust Error Logging
|
||||
|
||||
Complete integration of comprehensive error tracking for beta testing.
|
||||
"""
|
||||
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status, BackgroundTasks, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from loguru import logger
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
import tempfile
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
|
||||
from src.api.dependencies import get_db, optional_auth, get_storage
|
||||
from src.database.models import (
|
||||
User, Capture, Session as CaptureSession,
|
||||
CaptureMatch, Device, ManualIdentification
|
||||
)
|
||||
from src.parser.sub_parser import parse_sub_file
|
||||
from src.gps.validator import validate_gps_coordinates
|
||||
from src.matcher.engine import SignatureMatcher
|
||||
from src.logging.upload_logger import (
|
||||
log_upload_attempt,
|
||||
log_file_processing,
|
||||
log_parse_error,
|
||||
log_gps_validation_error,
|
||||
log_matching_results,
|
||||
log_manual_label,
|
||||
log_upload_complete,
|
||||
log_exception,
|
||||
log_performance_metrics,
|
||||
UploadStage
|
||||
)
|
||||
from config.settings import settings
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def generate_upload_id() -> str:
|
||||
"""Generate unique upload identifier"""
|
||||
return f"upload_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def extract_client_info(request: Request) -> dict:
|
||||
"""Extract client information from request"""
|
||||
return {
|
||||
"user_agent": request.headers.get("user-agent", "unknown"),
|
||||
"ip_address": request.client.host if request.client else "unknown",
|
||||
"referer": request.headers.get("referer"),
|
||||
"accept_language": request.headers.get("accept-language"),
|
||||
"content_length": request.headers.get("content-length")
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ENHANCED UPLOAD ENDPOINT WITH LOGGING
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/upload/beta")
|
||||
async def upload_captures_beta(
|
||||
request: Request,
|
||||
manifest: str = Form(..., description="JSON manifest with GPS data and manual labels"),
|
||||
files: List[UploadFile] = File(..., description=".sub files to upload"),
|
||||
background_tasks: BackgroundTasks = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth),
|
||||
storage = Depends(get_storage)
|
||||
):
|
||||
"""
|
||||
BETA: Enhanced upload with comprehensive error logging
|
||||
|
||||
Logs everything for debugging during beta testing:
|
||||
- Upload attempts and client info
|
||||
- Individual file processing stages
|
||||
- Parse errors with file samples
|
||||
- GPS validation failures
|
||||
- Matching results and performance
|
||||
- Manual labels
|
||||
- Performance metrics
|
||||
- Exceptions with full context
|
||||
"""
|
||||
|
||||
upload_start = time.time()
|
||||
upload_id = generate_upload_id()
|
||||
client_info = extract_client_info(request)
|
||||
|
||||
# Performance tracking
|
||||
perf_metrics = {}
|
||||
|
||||
try:
|
||||
# Parse manifest
|
||||
try:
|
||||
manifest_data = json.loads(manifest)
|
||||
except json.JSONDecodeError as e:
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.VALIDATION,
|
||||
exception=e,
|
||||
context={"manifest_sample": manifest[:200]}
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON manifest: {str(e)}"
|
||||
)
|
||||
|
||||
session_uuid = manifest_data.get("session_uuid")
|
||||
captures_manifest = manifest_data.get("captures", [])
|
||||
manual_labels_list = manifest_data.get("manual_labels", [])
|
||||
|
||||
# Calculate total upload size
|
||||
total_size = sum([file.size for file in files if file.size])
|
||||
|
||||
# Log upload attempt
|
||||
log_upload_attempt(
|
||||
upload_id=upload_id,
|
||||
user_id=user.id if user else None,
|
||||
session_id=session_uuid,
|
||||
file_count=len(files),
|
||||
total_size=total_size,
|
||||
client_info=client_info
|
||||
)
|
||||
|
||||
logger.bind(upload_id=upload_id).info(
|
||||
f"Processing upload: {len(files)} files, {total_size/1024/1024:.2f} MB"
|
||||
)
|
||||
|
||||
# Create manual labels lookup
|
||||
manual_labels_map = {label['filename']: label for label in manual_labels_list}
|
||||
|
||||
# Get or create session
|
||||
capture_session = None
|
||||
if session_uuid:
|
||||
capture_session = db.query(CaptureSession).filter_by(
|
||||
session_uuid=session_uuid
|
||||
).first()
|
||||
|
||||
if not capture_session:
|
||||
capture_session = CaptureSession(
|
||||
session_uuid=session_uuid,
|
||||
user_id=user.id if user else None,
|
||||
started_at=datetime.utcnow()
|
||||
)
|
||||
db.add(capture_session)
|
||||
db.flush()
|
||||
|
||||
# Create filename to manifest mapping
|
||||
manifest_map = {c['filename']: c for c in captures_manifest}
|
||||
|
||||
# Initialize matcher
|
||||
matcher = SignatureMatcher(db)
|
||||
|
||||
# Process files
|
||||
results = {"successful": [], "failed": [], "duplicates": []}
|
||||
file_perf_metrics = []
|
||||
|
||||
for file in files:
|
||||
file_start = time.time()
|
||||
file_metrics = {"filename": file.filename}
|
||||
|
||||
try:
|
||||
# =====================================================
|
||||
# STAGE 1: Read and Hash File
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
content = await file.read()
|
||||
file_hash = hashlib.sha256(content).hexdigest()
|
||||
|
||||
file_metrics["read_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# Check for duplicates
|
||||
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
|
||||
if existing:
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.VALIDATION,
|
||||
success=True,
|
||||
duration_ms=file_metrics["read_time_ms"],
|
||||
metadata={"status": "duplicate"}
|
||||
)
|
||||
|
||||
results["duplicates"].append({
|
||||
"filename": file.filename,
|
||||
"file_hash": file_hash
|
||||
})
|
||||
continue
|
||||
|
||||
# =====================================================
|
||||
# STAGE 2: Validate Manifest Entry
|
||||
# =====================================================
|
||||
manifest_entry = manifest_map.get(file.filename)
|
||||
if not manifest_entry:
|
||||
error_msg = "Missing manifest entry"
|
||||
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.VALIDATION,
|
||||
success=False,
|
||||
duration_ms=file_metrics["read_time_ms"],
|
||||
error=error_msg
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": error_msg
|
||||
})
|
||||
continue
|
||||
|
||||
# =====================================================
|
||||
# STAGE 3: Validate GPS
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
try:
|
||||
latitude = float(manifest_entry['latitude'])
|
||||
longitude = float(manifest_entry['longitude'])
|
||||
accuracy = manifest_entry.get('accuracy')
|
||||
|
||||
if not validate_gps_coordinates(latitude, longitude, accuracy):
|
||||
log_gps_validation_error(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
accuracy=accuracy,
|
||||
reason="Coordinates out of range or invalid"
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Invalid GPS: ({latitude}, {longitude})"
|
||||
})
|
||||
continue
|
||||
|
||||
file_metrics["gps_validation_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
except (ValueError, KeyError) as e:
|
||||
log_gps_validation_error(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
latitude=manifest_entry.get('latitude', 0),
|
||||
longitude=manifest_entry.get('longitude', 0),
|
||||
accuracy=manifest_entry.get('accuracy'),
|
||||
reason=f"Parse error: {str(e)}"
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"GPS parse error: {str(e)}"
|
||||
})
|
||||
continue
|
||||
|
||||
# =====================================================
|
||||
# STAGE 4: Parse .sub File
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix='.sub') as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
metadata = parse_sub_file(tmp_path)
|
||||
file_metrics["parse_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.PARSING,
|
||||
success=True,
|
||||
duration_ms=file_metrics["parse_time_ms"],
|
||||
metadata={
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol,
|
||||
"modulation": metadata.modulation,
|
||||
"format": metadata.file_format
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Detailed parse error logging
|
||||
content_str = content.decode('utf-8', errors='ignore')
|
||||
|
||||
log_parse_error(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_content_sample=content_str,
|
||||
error_type=type(e).__name__,
|
||||
error_message=str(e),
|
||||
traceback_str=traceback.format_exc(),
|
||||
file_hash=file_hash
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Parse failed: {str(e)}"
|
||||
})
|
||||
continue
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
# =====================================================
|
||||
# STAGE 5: Store File
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
storage_path = storage.save_file(
|
||||
file_hash=file_hash,
|
||||
content=content,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
file_metrics["storage_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# =====================================================
|
||||
# STAGE 6: Create Capture Record
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
|
||||
capture = Capture(
|
||||
file_hash=file_hash,
|
||||
session_id=capture_session.id if capture_session else None,
|
||||
user_id=user.id if user else None,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
altitude=manifest_entry.get('altitude'),
|
||||
gps_accuracy=accuracy,
|
||||
captured_at=datetime.fromisoformat(
|
||||
manifest_entry['timestamp'].replace('Z', '+00:00')
|
||||
),
|
||||
frequency=metadata.frequency,
|
||||
modulation=metadata.modulation,
|
||||
preset=metadata.preset,
|
||||
protocol=metadata.protocol,
|
||||
bit_length=metadata.bit_length,
|
||||
key_data=metadata.key_data,
|
||||
timing_element=metadata.timing_element,
|
||||
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
|
||||
raw_format=metadata.file_format,
|
||||
file_path=storage_path,
|
||||
file_size=len(content)
|
||||
)
|
||||
|
||||
db.add(capture)
|
||||
db.flush() # Get capture ID
|
||||
|
||||
file_metrics["db_insert_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# =====================================================
|
||||
# STAGE 7: Automatic Device Matching
|
||||
# =====================================================
|
||||
stage_start = time.time()
|
||||
matches = []
|
||||
matcher_stats = {}
|
||||
|
||||
try:
|
||||
match_results = matcher.match(metadata)
|
||||
matcher_stats = {
|
||||
"strategies_used": len(match_results),
|
||||
"total_candidates": len(match_results)
|
||||
}
|
||||
|
||||
# Store top 10 matches
|
||||
for match_result in match_results[:10]:
|
||||
device = db.query(Device).filter_by(id=match_result.device_id).first()
|
||||
if not device:
|
||||
continue
|
||||
|
||||
capture_match = CaptureMatch(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
confidence=match_result.confidence,
|
||||
match_method=match_result.method,
|
||||
match_details=match_result.details or {}
|
||||
)
|
||||
db.add(capture_match)
|
||||
|
||||
matches.append({
|
||||
"device_id": device.id,
|
||||
"manufacturer": device.manufacturer,
|
||||
"model": device.model,
|
||||
"device_type": device.device_type,
|
||||
"confidence": match_result.confidence,
|
||||
"method": match_result.method
|
||||
})
|
||||
|
||||
# Set best match
|
||||
if matches:
|
||||
capture.device_id = matches[0]["device_id"]
|
||||
capture.match_confidence = matches[0]["confidence"]
|
||||
capture.match_method = matches[0]["method"]
|
||||
|
||||
file_metrics["matching_time_ms"] = (time.time() - stage_start) * 1000
|
||||
|
||||
# Log matching results
|
||||
log_matching_results(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_hash=file_hash,
|
||||
matches=matches,
|
||||
duration_ms=file_metrics["matching_time_ms"],
|
||||
matcher_stats=matcher_stats
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.MATCHING,
|
||||
exception=e,
|
||||
context={"filename": file.filename, "file_hash": file_hash}
|
||||
)
|
||||
# Continue without matches
|
||||
|
||||
# =====================================================
|
||||
# STAGE 8: Manual Device Labeling
|
||||
# =====================================================
|
||||
manual_label = manual_labels_map.get(file.filename)
|
||||
if manual_label:
|
||||
try:
|
||||
await handle_manual_label_with_logging(
|
||||
upload_id=upload_id,
|
||||
db=db,
|
||||
capture=capture,
|
||||
manual_label=manual_label,
|
||||
user=user,
|
||||
file_hash=file_hash
|
||||
)
|
||||
except Exception as e:
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.DATABASE,
|
||||
exception=e,
|
||||
context={"manual_label": manual_label}
|
||||
)
|
||||
|
||||
# Commit this capture
|
||||
db.commit()
|
||||
|
||||
# Calculate total file processing time
|
||||
file_metrics["total_time_ms"] = (time.time() - file_start) * 1000
|
||||
file_perf_metrics.append(file_metrics)
|
||||
|
||||
# Log successful processing
|
||||
log_file_processing(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
file_size=len(content),
|
||||
file_hash=file_hash,
|
||||
stage=UploadStage.COMPLETE,
|
||||
success=True,
|
||||
duration_ms=file_metrics["total_time_ms"],
|
||||
metadata={
|
||||
"matches_found": len(matches),
|
||||
"manual_label": manual_label is not None
|
||||
}
|
||||
)
|
||||
|
||||
results["successful"].append({
|
||||
"filename": file.filename,
|
||||
"file_hash": file_hash,
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol,
|
||||
"matches": matches,
|
||||
"manual_label": manual_label is not None,
|
||||
"processing_time_ms": round(file_metrics["total_time_ms"], 2)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
# Catch-all for unexpected errors
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.DATABASE,
|
||||
exception=e,
|
||||
context={
|
||||
"filename": file.filename,
|
||||
"user_id": user.id if user else None
|
||||
}
|
||||
)
|
||||
|
||||
results["failed"].append({
|
||||
"filename": file.filename,
|
||||
"error": f"Unexpected error: {str(e)}"
|
||||
})
|
||||
|
||||
# =====================================================
|
||||
# Upload Complete - Log Summary
|
||||
# =====================================================
|
||||
total_duration_ms = (time.time() - upload_start) * 1000
|
||||
|
||||
# Calculate aggregate performance metrics
|
||||
if file_perf_metrics:
|
||||
avg_metrics = {
|
||||
"avg_parse_time_ms": sum(m.get("parse_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"avg_matching_time_ms": sum(m.get("matching_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"avg_storage_time_ms": sum(m.get("storage_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"avg_total_time_ms": sum(m.get("total_time_ms", 0) for m in file_perf_metrics) / len(file_perf_metrics),
|
||||
"total_upload_time_ms": total_duration_ms
|
||||
}
|
||||
|
||||
log_performance_metrics(
|
||||
upload_id=upload_id,
|
||||
metrics=avg_metrics
|
||||
)
|
||||
|
||||
log_upload_complete(
|
||||
upload_id=upload_id,
|
||||
total_duration_ms=total_duration_ms,
|
||||
successful_count=len(results["successful"]),
|
||||
failed_count=len(results["failed"]),
|
||||
duplicate_count=len(results["duplicates"]),
|
||||
summary={
|
||||
"total_files": len(files),
|
||||
"total_size_mb": round(total_size / 1024 / 1024, 2),
|
||||
"user_id": user.id if user else None,
|
||||
"session_uuid": session_uuid
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"upload_id": upload_id,
|
||||
"successful": results["successful"],
|
||||
"failed": results["failed"],
|
||||
"duplicates": results["duplicates"],
|
||||
"summary": {
|
||||
"total_files": len(files),
|
||||
"successful": len(results["successful"]),
|
||||
"failed": len(results["failed"]),
|
||||
"duplicates": len(results["duplicates"]),
|
||||
"processing_time_sec": round(total_duration_ms / 1000, 2)
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# Top-level exception handler
|
||||
log_exception(
|
||||
upload_id=upload_id,
|
||||
stage=UploadStage.VALIDATION,
|
||||
exception=e,
|
||||
context={
|
||||
"user_id": user.id if user else None,
|
||||
"file_count": len(files)
|
||||
}
|
||||
)
|
||||
|
||||
logger.error(f"Upload failed completely: {str(e)}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Upload failed: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
async def handle_manual_label_with_logging(
|
||||
upload_id: str,
|
||||
db: Session,
|
||||
capture: Capture,
|
||||
manual_label: dict,
|
||||
user: Optional[User],
|
||||
file_hash: str
|
||||
):
|
||||
"""Process manual label with logging"""
|
||||
|
||||
source = manual_label.get("source", "manual")
|
||||
|
||||
if source == "manual_existing":
|
||||
device_id = manual_label.get("device_id")
|
||||
if not device_id:
|
||||
return
|
||||
|
||||
device = db.query(Device).filter_by(id=device_id).first()
|
||||
if not device:
|
||||
logger.warning(f"Manual label references non-existent device: {device_id}")
|
||||
return
|
||||
|
||||
capture.device_id = device_id
|
||||
capture.match_confidence = 1.0
|
||||
capture.match_method = "manual_selection"
|
||||
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device_id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False,
|
||||
verification_count=1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
log_manual_label(
|
||||
upload_id=upload_id,
|
||||
filename=capture.file_path.split('/')[-1],
|
||||
file_hash=file_hash,
|
||||
label_type="existing_device",
|
||||
device_id=device_id,
|
||||
device_info={
|
||||
"manufacturer": device.manufacturer,
|
||||
"model": device.model,
|
||||
"device_type": device.device_type
|
||||
},
|
||||
user_id=user.id if user else None
|
||||
)
|
||||
|
||||
elif source == "manual_new":
|
||||
device_type = manual_label.get("device_type")
|
||||
manufacturer = manual_label.get("manufacturer")
|
||||
model = manual_label.get("model")
|
||||
|
||||
if not all([device_type, manufacturer, model]):
|
||||
return
|
||||
|
||||
# Check if device already exists
|
||||
existing_device = db.query(Device).filter_by(
|
||||
manufacturer=manufacturer,
|
||||
model=model
|
||||
).first()
|
||||
|
||||
if existing_device:
|
||||
device = existing_device
|
||||
else:
|
||||
device = Device(
|
||||
manufacturer=manufacturer,
|
||||
model=model,
|
||||
device_type=device_type,
|
||||
typical_frequency=capture.frequency,
|
||||
protocol=capture.protocol,
|
||||
source="community",
|
||||
verified=False,
|
||||
metadata_json={"submitter_user_id": user.id if user else None}
|
||||
)
|
||||
db.add(device)
|
||||
db.flush()
|
||||
|
||||
capture.device_id = device.id
|
||||
capture.match_confidence = 1.0
|
||||
capture.match_method = "manual_new_device"
|
||||
|
||||
manual_id = ManualIdentification(
|
||||
capture_id=capture.id,
|
||||
device_id=device.id,
|
||||
user_id=user.id if user else None,
|
||||
confidence=1.0,
|
||||
notes=manual_label.get("notes"),
|
||||
verified=False,
|
||||
verification_count=1
|
||||
)
|
||||
db.add(manual_id)
|
||||
|
||||
log_manual_label(
|
||||
upload_id=upload_id,
|
||||
filename=capture.file_path.split('/')[-1],
|
||||
file_hash=file_hash,
|
||||
label_type="new_device",
|
||||
device_id=device.id,
|
||||
device_info={
|
||||
"manufacturer": manufacturer,
|
||||
"model": model,
|
||||
"device_type": device_type
|
||||
},
|
||||
user_id=user.id if user else None
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLIENT ERROR REPORTING ENDPOINT
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/report_client_error")
|
||||
async def report_client_error(
|
||||
request: Request,
|
||||
error_data: dict,
|
||||
user: User = Depends(optional_auth)
|
||||
):
|
||||
"""
|
||||
Endpoint for client-side JavaScript to report errors
|
||||
|
||||
Expected payload:
|
||||
{
|
||||
"upload_id": "upload_20260115_...",
|
||||
"error_type": "ParseError",
|
||||
"error_message": "Failed to read .sub file",
|
||||
"stack_trace": "...",
|
||||
"filename": "capture.sub",
|
||||
"browser_info": {...}
|
||||
}
|
||||
"""
|
||||
from src.logging.upload_logger import log_client_error
|
||||
|
||||
upload_id = error_data.get("upload_id", "unknown")
|
||||
client_info = extract_client_info(request)
|
||||
client_info.update(error_data.get("browser_info", {}))
|
||||
|
||||
log_client_error(
|
||||
upload_id=upload_id,
|
||||
error_type=error_data.get("error_type", "UnknownError"),
|
||||
error_message=error_data.get("error_message", "No message"),
|
||||
client_info=client_info,
|
||||
stack_trace=error_data.get("stack_trace")
|
||||
)
|
||||
|
||||
return {"status": "logged"}
|
||||
Reference in New Issue
Block a user