feat: Add wardriving data import system and cleanup tools
Major Features: - Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS) - Data source tracking (test/mock/production) - Database cleanup tools for managing test data - JSON file persistence for simplified server - UI improvements for map controls Backend: - Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter - Added data_source and session_id tracking to captures - New API endpoints: DELETE /api/v1/admin/cleanup - Auto-save/load functionality for captures_simple.json - Updated stats endpoint to show data source breakdown CLI Tools: - scripts/import_wardriving_data.py - Batch import with --data-source flag - scripts/cleanup_database.py - Clean by source, session, or all - scripts/create_test_dataset.py - Generate GPS-tagged test data - scripts/test_wardriving_import.py - Test suite for importers Frontend: - Fixed map controls z-index and positioning issues - Moved controls to top-right to avoid zoom button overlap - Fixed Leaflet zoom controls rendering over header - Changed controls to position:fixed for persistent visibility - Export map object to window.map for proper invalidateSize Documentation: - docs/WARDRIVING_IMPORT.md - Complete import guide - docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation - docs/TEST_LOCATIONS.md - Test GPS coordinates reference - TEST_RESULTS_SUMMARY.md - Format testing results Test Data: - Created test_dataset_gps with 15 captures across 5 US cities - All test data marked with data_source="test" for easy cleanup Testing: - Verified Wigle CSV import (1 capture from West LA) - Verified GPS filename import (3 captures from UCLA area) - Verified GPS JSON companion files (15 captures, 5 cities) - Verified cleanup functionality (deleted 15 test captures) - Verified data persistence across server restarts 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,333 @@
|
||||
"""
|
||||
Wardriving Data Importer
|
||||
|
||||
Imports GPS-tagged SubGhz captures from various formats:
|
||||
- Wigle-WiFi CSV format (adapted for SubGhz)
|
||||
- GPS coordinate JSON files
|
||||
- Batch .sub files with companion GPS data
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from dataclasses import dataclass
|
||||
from loguru import logger
|
||||
|
||||
from .sub_parser import SubFileParser
|
||||
from .gps_extractor import GPSFilenameExtractor, GPSCoordinates
|
||||
|
||||
|
||||
@dataclass
|
||||
class WardrivingCapture:
|
||||
"""Single wardriving capture with GPS and RF data"""
|
||||
filename: str
|
||||
frequency: int # Hz
|
||||
protocol: str
|
||||
latitude: float
|
||||
longitude: float
|
||||
altitude: Optional[float] = None
|
||||
accuracy: Optional[float] = None
|
||||
timestamp: Optional[str] = None
|
||||
rssi: Optional[int] = None
|
||||
session_id: Optional[str] = None
|
||||
gps_source: str = "wardriving"
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
"""Convert to dictionary for API"""
|
||||
return {
|
||||
'filename': self.filename,
|
||||
'frequency': self.frequency,
|
||||
'protocol': self.protocol,
|
||||
'latitude': self.latitude,
|
||||
'longitude': self.longitude,
|
||||
'altitude': self.altitude,
|
||||
'accuracy': self.accuracy,
|
||||
'timestamp': self.timestamp,
|
||||
'rssi': self.rssi,
|
||||
'session_id': self.session_id,
|
||||
'gps_source': self.gps_source
|
||||
}
|
||||
|
||||
|
||||
class WigleCSVImporter:
|
||||
"""
|
||||
Import SubGhz wardriving data from Wigle-WiFi CSV format
|
||||
|
||||
Format:
|
||||
WigleWifi-1.4,appRelease=SubGHz-Wardrive-1.0,...
|
||||
SignalHash,Frequency(MHz),Protocol,RSSI(dBm),Latitude,Longitude,Altitude,Accuracy,...
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.sub_parser = SubFileParser()
|
||||
self.captures: List[WardrivingCapture] = []
|
||||
|
||||
def parse_csv(self, csv_path: str) -> List[WardrivingCapture]:
|
||||
"""Parse Wigle CSV file"""
|
||||
path = Path(csv_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"CSV file not found: {csv_path}")
|
||||
|
||||
self.captures = []
|
||||
|
||||
with open(path, 'r') as f:
|
||||
# Read header line
|
||||
header_line = f.readline().strip()
|
||||
if not header_line.startswith('WigleWifi'):
|
||||
raise ValueError("Not a valid Wigle CSV file")
|
||||
|
||||
# Read column headers
|
||||
reader = csv.DictReader(f)
|
||||
|
||||
for row in reader:
|
||||
try:
|
||||
capture = self._parse_row(row, path.parent)
|
||||
if capture:
|
||||
self.captures.append(capture)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse row: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Parsed {len(self.captures)} captures from {csv_path}")
|
||||
return self.captures
|
||||
|
||||
def _parse_row(self, row: Dict, base_dir: Path) -> Optional[WardrivingCapture]:
|
||||
"""Parse single CSV row"""
|
||||
# Get frequency in Hz
|
||||
freq_mhz = float(row.get('Frequency(MHz)', 0))
|
||||
frequency = int(freq_mhz * 1e6)
|
||||
|
||||
# Get GPS coordinates
|
||||
lat = float(row.get('Latitude', 0))
|
||||
lon = float(row.get('Longitude', 0))
|
||||
|
||||
if lat == 0 or lon == 0:
|
||||
return None
|
||||
|
||||
# Get .sub filename
|
||||
filename = row.get('BruceFile', '')
|
||||
if not filename:
|
||||
filename = f"capture_{row.get('SignalHash', 'unknown')}.sub"
|
||||
|
||||
# Parse altitude and accuracy if available
|
||||
altitude = None
|
||||
if row.get('Altitude'):
|
||||
try:
|
||||
altitude = float(row['Altitude'])
|
||||
except:
|
||||
pass
|
||||
|
||||
accuracy = None
|
||||
if row.get('Accuracy'):
|
||||
try:
|
||||
accuracy = float(row['Accuracy'])
|
||||
except:
|
||||
pass
|
||||
|
||||
# Get RSSI
|
||||
rssi = None
|
||||
if row.get('RSSI(dBm)'):
|
||||
try:
|
||||
rssi = int(row['RSSI(dBm)'])
|
||||
except:
|
||||
pass
|
||||
|
||||
return WardrivingCapture(
|
||||
filename=filename,
|
||||
frequency=frequency,
|
||||
protocol=row.get('Protocol', 'Unknown'),
|
||||
latitude=lat,
|
||||
longitude=lon,
|
||||
altitude=altitude,
|
||||
accuracy=accuracy,
|
||||
timestamp=row.get('FirstSeen', datetime.now().isoformat()),
|
||||
rssi=rssi,
|
||||
session_id=row.get('SessionID'),
|
||||
gps_source='wigle_csv'
|
||||
)
|
||||
|
||||
|
||||
class GPSJSONImporter:
|
||||
"""
|
||||
Import GPS coordinates from JSON files
|
||||
|
||||
Format:
|
||||
{
|
||||
"type": "gps_coordinate",
|
||||
"data": {
|
||||
"latitude": 34.0522,
|
||||
"longitude": -118.2437,
|
||||
"accuracy": 5.0,
|
||||
"timestamp": "2026-01-09T21:26:51Z"
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def parse_json(json_path: str) -> Optional[GPSCoordinates]:
|
||||
"""Parse GPS JSON file"""
|
||||
path = Path(json_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"JSON file not found: {json_path}")
|
||||
|
||||
with open(path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Extract GPS data
|
||||
gps_data = data.get('data', {})
|
||||
|
||||
lat = gps_data.get('latitude')
|
||||
lon = gps_data.get('longitude')
|
||||
|
||||
if lat is None or lon is None:
|
||||
return None
|
||||
|
||||
return GPSCoordinates(
|
||||
latitude=float(lat),
|
||||
longitude=float(lon),
|
||||
source='gps_json',
|
||||
accuracy=gps_data.get('accuracy'),
|
||||
altitude=gps_data.get('altitude')
|
||||
)
|
||||
|
||||
|
||||
class BatchImporter:
|
||||
"""
|
||||
Batch import .sub files with companion GPS data
|
||||
|
||||
Supports:
|
||||
1. CSV manifest (Wigle format)
|
||||
2. Individual JSON GPS files (one per .sub file)
|
||||
3. GPS in filenames
|
||||
4. Single JSON with multiple GPS points
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.sub_parser = SubFileParser()
|
||||
self.gps_extractor = GPSFilenameExtractor()
|
||||
self.csv_importer = WigleCSVImporter()
|
||||
self.captures: List[WardrivingCapture] = []
|
||||
|
||||
def import_from_directory(
|
||||
self,
|
||||
directory: str,
|
||||
csv_manifest: Optional[str] = None
|
||||
) -> List[WardrivingCapture]:
|
||||
"""
|
||||
Import all .sub files from directory
|
||||
|
||||
Priority:
|
||||
1. CSV manifest (if provided)
|
||||
2. Companion JSON files (filename.json for filename.sub)
|
||||
3. GPS in filename
|
||||
4. Skip file if no GPS found
|
||||
"""
|
||||
dir_path = Path(directory)
|
||||
if not dir_path.exists():
|
||||
raise FileNotFoundError(f"Directory not found: {directory}")
|
||||
|
||||
self.captures = []
|
||||
|
||||
# Method 1: CSV manifest
|
||||
if csv_manifest:
|
||||
csv_captures = self.csv_importer.parse_csv(csv_manifest)
|
||||
self.captures.extend(csv_captures)
|
||||
logger.info(f"Imported {len(csv_captures)} captures from CSV manifest")
|
||||
return self.captures
|
||||
|
||||
# Method 2: Process individual .sub files
|
||||
sub_files = list(dir_path.glob('*.sub'))
|
||||
logger.info(f"Found {len(sub_files)} .sub files")
|
||||
|
||||
for sub_file in sub_files:
|
||||
capture = self._import_sub_file(sub_file)
|
||||
if capture:
|
||||
self.captures.append(capture)
|
||||
|
||||
logger.info(f"Successfully imported {len(self.captures)} captures")
|
||||
return self.captures
|
||||
|
||||
def _import_sub_file(self, sub_path: Path) -> Optional[WardrivingCapture]:
|
||||
"""Import single .sub file with GPS from various sources"""
|
||||
|
||||
# Try to get GPS coordinates
|
||||
gps_coords = None
|
||||
|
||||
# 1. Check for companion JSON file
|
||||
json_path = sub_path.with_suffix('.json')
|
||||
if json_path.exists():
|
||||
gps_coords = GPSJSONImporter.parse_json(str(json_path))
|
||||
if gps_coords:
|
||||
gps_coords.source = 'companion_json'
|
||||
|
||||
# 2. Extract from filename
|
||||
if not gps_coords:
|
||||
gps_coords = self.gps_extractor.extract(sub_path.name)
|
||||
|
||||
# Skip if no GPS found
|
||||
if not gps_coords:
|
||||
logger.debug(f"No GPS for {sub_path.name}, skipping")
|
||||
return None
|
||||
|
||||
# Parse .sub file for RF metadata
|
||||
try:
|
||||
metadata = self.sub_parser.parse(str(sub_path))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse {sub_path.name}: {e}")
|
||||
return None
|
||||
|
||||
return WardrivingCapture(
|
||||
filename=sub_path.name,
|
||||
frequency=metadata.frequency if hasattr(metadata, 'frequency') else 0,
|
||||
protocol=metadata.protocol if hasattr(metadata, 'protocol') else 'Unknown',
|
||||
latitude=gps_coords.latitude,
|
||||
longitude=gps_coords.longitude,
|
||||
altitude=gps_coords.altitude,
|
||||
accuracy=gps_coords.accuracy,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
gps_source=gps_coords.source
|
||||
)
|
||||
|
||||
def to_upload_manifest(self) -> Dict:
|
||||
"""Convert captures to upload manifest format"""
|
||||
return {
|
||||
"session_uuid": f"batch_import_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
|
||||
"captures": [c.to_dict() for c in self.captures]
|
||||
}
|
||||
|
||||
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Test Wigle CSV import
|
||||
print("=" * 60)
|
||||
print("Testing Wigle CSV Import")
|
||||
print("=" * 60)
|
||||
|
||||
csv_importer = WigleCSVImporter()
|
||||
csv_path = "signatures/t-embed-rf/test_export_wigle.csv"
|
||||
|
||||
if Path(csv_path).exists():
|
||||
captures = csv_importer.parse_csv(csv_path)
|
||||
print(f"\nImported {len(captures)} captures from CSV")
|
||||
|
||||
for capture in captures:
|
||||
print(f"\n File: {capture.filename}")
|
||||
print(f" Frequency: {capture.frequency / 1e6:.2f} MHz")
|
||||
print(f" GPS: {capture.latitude}, {capture.longitude}")
|
||||
print(f" Accuracy: {capture.accuracy}m")
|
||||
|
||||
# Test batch import from directory
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing Batch Import")
|
||||
print("=" * 60)
|
||||
|
||||
batch_importer = BatchImporter()
|
||||
captures = batch_importer.import_from_directory("signatures/t-embed-rf")
|
||||
|
||||
print(f"\nImported {len(captures)} total captures")
|
||||
print(f"\nGPS Sources:")
|
||||
for source in set(c.gps_source for c in captures):
|
||||
count = sum(1 for c in captures if c.gps_source == source)
|
||||
print(f" {source}: {count}")
|
||||
Reference in New Issue
Block a user