Phase 3 Complete: Web Interface MVP
Major Achievements: - ✅ Full web interface (1,520+ lines of frontend code) - ✅ Interactive Leaflet.js map with marker clustering - ✅ Drag-and-drop upload system with GPS input - ✅ Search & filter UI with multi-criteria - ✅ Statistics dashboard with Chart.js - ✅ Responsive mobile-friendly design Backend: - ✅ FastAPI static file serving - ✅ Simplified server mode (main_simple.py) - ✅ Improved startup script with port auto-selection - ✅ PostgreSQL schema ready (requires setup) Database: - ✅ SQLite populated with 85 Flipper Zero signatures - ✅ Device matching system operational - ✅ Frequency-based search working Documentation: - ✅ PHASE_3_COMPLETE.md - Technical summary - ✅ WEB_INTERFACE_README.md - User guide - ✅ WEBAPP_STARTUP_GUIDE.md - Troubleshooting - ✅ POSTGRESQL_SETUP_EXPLANATION.md - DB setup guide - ✅ DATABASE_POPULATION_SUCCESS.md - Import report - ✅ DEVICE_IDENTIFICATION_REPORT.md - Matching analysis Files Created: - templates/index.html (260 lines) - static/css/main.css (500 lines) - static/js/*.js (760 lines total) - src/api/main_simple.py (simplified server) - start_web.sh (auto port selection) Status: Production MVP Ready Next: Phase 4 - API & Integration 🛰️ Generated with Claude Code https://claude.com/claude-code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Executable
+329
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import T-Embed RF captures as signature database entries
|
||||
|
||||
This script:
|
||||
1. Scans T-Embed .sub files
|
||||
2. Extracts RF signal patterns
|
||||
3. Creates device and signature records
|
||||
4. Populates database for matching
|
||||
|
||||
Based on real wardriving captures from T-Embed device
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
from src.database.models import Device, Signature, FlipperSignature, Base
|
||||
from src.database.connection import get_engine, get_session
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
||||
class TembedSignatureImporter:
|
||||
"""Import T-Embed RF captures as device signatures"""
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self.session = session
|
||||
self.parser = SubFileParser()
|
||||
self.stats = {
|
||||
'files_found': 0,
|
||||
'files_parsed': 0,
|
||||
'files_skipped': 0,
|
||||
'devices_created': 0,
|
||||
'signatures_created': 0,
|
||||
'errors': []
|
||||
}
|
||||
|
||||
def import_directory(self, directory: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
Import all .sub files from directory
|
||||
|
||||
Args:
|
||||
directory: Path to directory containing .sub files
|
||||
|
||||
Returns:
|
||||
Statistics dictionary
|
||||
"""
|
||||
print(f"Scanning {directory} for .sub files...")
|
||||
|
||||
# Find all .sub files
|
||||
sub_files = list(directory.glob('*.sub'))
|
||||
self.stats['files_found'] = len(sub_files)
|
||||
|
||||
print(f"Found {len(sub_files)} .sub files\n")
|
||||
|
||||
for sub_file in sorted(sub_files):
|
||||
print(f"Processing: {sub_file.name}")
|
||||
try:
|
||||
self._import_file(sub_file)
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing {sub_file.name}: {e}"
|
||||
print(f" ❌ {error_msg}")
|
||||
self.stats['errors'].append(error_msg)
|
||||
|
||||
# Commit all changes
|
||||
try:
|
||||
self.session.commit()
|
||||
print("\n✅ Database changes committed")
|
||||
except Exception as e:
|
||||
self.session.rollback()
|
||||
print(f"\n❌ Failed to commit: {e}")
|
||||
self.stats['errors'].append(f"Commit failed: {e}")
|
||||
|
||||
return self.stats
|
||||
|
||||
def _import_file(self, file_path: Path):
|
||||
"""Import a single .sub file"""
|
||||
|
||||
# Parse file
|
||||
try:
|
||||
metadata = self.parser.parse(str(file_path))
|
||||
except Exception as e:
|
||||
self.stats['files_skipped'] += 1
|
||||
raise ValueError(f"Parse failed: {e}")
|
||||
|
||||
# Skip if empty (frequency = 0, no data)
|
||||
if metadata.frequency == 0 or (metadata.file_format == 'RAW' and not metadata.raw_data):
|
||||
print(f" ⏭️ Skipped: Empty capture")
|
||||
self.stats['files_skipped'] += 1
|
||||
return
|
||||
|
||||
self.stats['files_parsed'] += 1
|
||||
|
||||
# Load GPS data if available
|
||||
gps_data = self._load_gps_data(file_path)
|
||||
|
||||
# Create device record
|
||||
device = self._create_device(metadata, file_path, gps_data)
|
||||
|
||||
# Create signature record
|
||||
signature = self._create_signature(metadata, device)
|
||||
|
||||
# Create Flipper signature record (for compatibility)
|
||||
flipper_sig = self._create_flipper_signature(metadata, device, file_path)
|
||||
|
||||
print(f" ✅ Device: {device.device_name}")
|
||||
print(f" Frequency: {metadata.frequency/1e6:.2f} MHz")
|
||||
if metadata.raw_data:
|
||||
print(f" RAW samples: {len(metadata.raw_data)}")
|
||||
if gps_data:
|
||||
print(f" GPS: {gps_data['data']['latitude']:.4f}, {gps_data['data']['longitude']:.4f}")
|
||||
|
||||
def _load_gps_data(self, sub_file: Path) -> Optional[Dict]:
|
||||
"""Load GPS coordinates for a .sub file if available"""
|
||||
|
||||
# Look for matching GPS JSON files in same directory
|
||||
# Pattern: gps_coordinates_YYYYMMDD_HHMMSS.json
|
||||
gps_files = sorted(sub_file.parent.glob('gps_coordinates_*.json'))
|
||||
|
||||
if not gps_files:
|
||||
return None
|
||||
|
||||
# Use the most recent GPS file (simple heuristic)
|
||||
gps_file = gps_files[-1]
|
||||
|
||||
try:
|
||||
with open(gps_file, 'r') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Could not load GPS data: {e}")
|
||||
return None
|
||||
|
||||
def _create_device(self, metadata, file_path: Path, gps_data: Optional[Dict]) -> Device:
|
||||
"""Create a Device record from metadata"""
|
||||
|
||||
# Generate device name from file and frequency
|
||||
device_name = self._generate_device_name(file_path, metadata)
|
||||
|
||||
# Determine device type from frequency
|
||||
device_type = self._guess_device_type(metadata.frequency)
|
||||
|
||||
# Check if device already exists
|
||||
existing = self.session.query(Device).filter(
|
||||
Device.device_name == device_name
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
print(f" ℹ️ Device already exists: {device_name}")
|
||||
return existing
|
||||
|
||||
# Create new device
|
||||
device = Device(
|
||||
device_name=device_name,
|
||||
manufacturer='Unknown',
|
||||
model='T-Embed Capture',
|
||||
device_type=device_type,
|
||||
typical_frequency=metadata.frequency,
|
||||
protocol=metadata.protocol if metadata.protocol else 'RAW',
|
||||
description=f"Captured from T-Embed RF device at {file_path.name}",
|
||||
first_seen=datetime.utcnow(),
|
||||
is_verified=False
|
||||
)
|
||||
|
||||
self.session.add(device)
|
||||
self.session.flush() # Get device.id
|
||||
|
||||
self.stats['devices_created'] += 1
|
||||
|
||||
return device
|
||||
|
||||
def _create_signature(self, metadata, device: Device) -> Signature:
|
||||
"""Create a Signature record for pattern matching"""
|
||||
|
||||
# Check if signature already exists
|
||||
existing = self.session.query(Signature).filter(
|
||||
Signature.device_id == device.id,
|
||||
Signature.frequency == metadata.frequency
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
# Extract timing patterns for RAW format
|
||||
timing_min, timing_max = None, None
|
||||
if metadata.raw_data and len(metadata.raw_data) > 0:
|
||||
# Use absolute values for timing
|
||||
abs_timings = [abs(t) for t in metadata.raw_data]
|
||||
timing_min = min(abs_timings)
|
||||
timing_max = max(abs_timings)
|
||||
|
||||
signature = Signature(
|
||||
device_id=device.id,
|
||||
protocol=metadata.protocol if metadata.protocol else 'RAW',
|
||||
frequency=metadata.frequency,
|
||||
modulation=metadata.modulation,
|
||||
bit_pattern=None, # Not available for RAW
|
||||
bit_mask=None,
|
||||
timing_min=timing_min,
|
||||
timing_max=timing_max,
|
||||
raw_pattern=','.join(map(str, metadata.raw_data)) if metadata.raw_data else None,
|
||||
confidence_threshold=0.7, # Default threshold
|
||||
source='tembed_wardriving',
|
||||
created_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
self.session.add(signature)
|
||||
self.stats['signatures_created'] += 1
|
||||
|
||||
return signature
|
||||
|
||||
def _create_flipper_signature(self, metadata, device: Device, file_path: Path) -> FlipperSignature:
|
||||
"""Create FlipperSignature record for compatibility"""
|
||||
|
||||
# Check if exists
|
||||
existing = self.session.query(FlipperSignature).filter(
|
||||
FlipperSignature.device_id == device.id
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
flipper_sig = FlipperSignature(
|
||||
device_id=device.id,
|
||||
frequency=metadata.frequency,
|
||||
preset=metadata.preset if metadata.preset else '0',
|
||||
protocol=metadata.protocol if metadata.protocol else 'RAW',
|
||||
bit=metadata.bit_length,
|
||||
key=metadata.key_data,
|
||||
te=metadata.timing_element,
|
||||
raw_data=','.join(map(str, metadata.raw_data)) if metadata.raw_data else None,
|
||||
source_file=file_path.name,
|
||||
imported_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
self.session.add(flipper_sig)
|
||||
|
||||
return flipper_sig
|
||||
|
||||
def _generate_device_name(self, file_path: Path, metadata) -> str:
|
||||
"""Generate a unique device name"""
|
||||
freq_mhz = metadata.frequency / 1e6
|
||||
return f"{file_path.stem}_{freq_mhz:.0f}MHz"
|
||||
|
||||
def _guess_device_type(self, frequency: int) -> str:
|
||||
"""Guess device type from frequency"""
|
||||
freq_mhz = frequency / 1e6
|
||||
|
||||
if 300 <= freq_mhz <= 350:
|
||||
return 'garage_door'
|
||||
elif 400 <= freq_mhz <= 440:
|
||||
return 'remote_control'
|
||||
elif 860 <= freq_mhz <= 870:
|
||||
return 'sensor'
|
||||
elif 900 <= freq_mhz <= 930:
|
||||
return 'ism_device' # 915 MHz ISM band
|
||||
else:
|
||||
return 'unknown'
|
||||
|
||||
def print_summary(self):
|
||||
"""Print import summary"""
|
||||
print("\n" + "="*60)
|
||||
print("IMPORT SUMMARY")
|
||||
print("="*60)
|
||||
print(f"Files found: {self.stats['files_found']}")
|
||||
print(f"Files parsed: {self.stats['files_parsed']}")
|
||||
print(f"Files skipped: {self.stats['files_skipped']}")
|
||||
print(f"Devices created: {self.stats['devices_created']}")
|
||||
print(f"Signatures created: {self.stats['signatures_created']}")
|
||||
|
||||
if self.stats['errors']:
|
||||
print(f"\nErrors: {len(self.stats['errors'])}")
|
||||
for error in self.stats['errors']:
|
||||
print(f" - {error}")
|
||||
else:
|
||||
print("\n✅ No errors")
|
||||
|
||||
print("="*60)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*60)
|
||||
print("T-Embed RF Signature Importer")
|
||||
print("="*60)
|
||||
print()
|
||||
|
||||
# Get database session
|
||||
try:
|
||||
engine = get_engine()
|
||||
session = get_session()
|
||||
print("✅ Database connected")
|
||||
except Exception as e:
|
||||
print(f"❌ Database connection failed: {e}")
|
||||
print("\nMake sure PostgreSQL is running and configured correctly")
|
||||
return 1
|
||||
|
||||
# Create tables if needed
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
print("✅ Database tables ready\n")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not create tables: {e}\n")
|
||||
|
||||
# Find T-Embed directory
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
|
||||
if not tembed_dir.exists():
|
||||
print(f"❌ Directory not found: {tembed_dir}")
|
||||
return 1
|
||||
|
||||
# Import signatures
|
||||
importer = TembedSignatureImporter(session)
|
||||
stats = importer.import_directory(tembed_dir)
|
||||
importer.print_summary()
|
||||
|
||||
session.close()
|
||||
|
||||
return 0 if not stats['errors'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user