feat: Add wardriving data import system and cleanup tools
Major Features: - Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS) - Data source tracking (test/mock/production) - Database cleanup tools for managing test data - JSON file persistence for simplified server - UI improvements for map controls Backend: - Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter - Added data_source and session_id tracking to captures - New API endpoints: DELETE /api/v1/admin/cleanup - Auto-save/load functionality for captures_simple.json - Updated stats endpoint to show data source breakdown CLI Tools: - scripts/import_wardriving_data.py - Batch import with --data-source flag - scripts/cleanup_database.py - Clean by source, session, or all - scripts/create_test_dataset.py - Generate GPS-tagged test data - scripts/test_wardriving_import.py - Test suite for importers Frontend: - Fixed map controls z-index and positioning issues - Moved controls to top-right to avoid zoom button overlap - Fixed Leaflet zoom controls rendering over header - Changed controls to position:fixed for persistent visibility - Export map object to window.map for proper invalidateSize Documentation: - docs/WARDRIVING_IMPORT.md - Complete import guide - docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation - docs/TEST_LOCATIONS.md - Test GPS coordinates reference - TEST_RESULTS_SUMMARY.md - Format testing results Test Data: - Created test_dataset_gps with 15 captures across 5 US cities - All test data marked with data_source="test" for easy cleanup Testing: - Verified Wigle CSV import (1 capture from West LA) - Verified GPS filename import (3 captures from UCLA area) - Verified GPS JSON companion files (15 captures, 5 cities) - Verified cleanup functionality (deleted 15 test captures) - Verified data persistence across server restarts 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from datetime import datetime
|
||||
from fastapi import FastAPI, Request, UploadFile, File, Form
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
@@ -38,11 +39,54 @@ BASE_DIR = Path(__file__).parent.parent.parent
|
||||
app.mount("/static", StaticFiles(directory=str(BASE_DIR / "static")), name="static")
|
||||
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
|
||||
|
||||
# Persistent storage file
|
||||
STORAGE_FILE = BASE_DIR / "data" / "captures_simple.json"
|
||||
STORAGE_FILE.parent.mkdir(exist_ok=True)
|
||||
|
||||
# In-memory storage for uploaded captures (simplified mode)
|
||||
captures_storage = []
|
||||
upload_counter = 0
|
||||
|
||||
|
||||
def load_captures():
|
||||
"""Load captures from JSON file"""
|
||||
global captures_storage, upload_counter
|
||||
|
||||
if STORAGE_FILE.exists():
|
||||
try:
|
||||
with open(STORAGE_FILE, 'r') as f:
|
||||
data = json.load(f)
|
||||
captures_storage = data.get('captures', [])
|
||||
upload_counter = data.get('upload_counter', 0)
|
||||
print(f"✅ Loaded {len(captures_storage)} captures from {STORAGE_FILE}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to load captures: {e}")
|
||||
captures_storage = []
|
||||
upload_counter = 0
|
||||
else:
|
||||
print(f"ℹ️ No existing captures file at {STORAGE_FILE}")
|
||||
|
||||
|
||||
def save_captures():
|
||||
"""Save captures to JSON file"""
|
||||
try:
|
||||
data = {
|
||||
'captures': captures_storage,
|
||||
'upload_counter': upload_counter,
|
||||
'last_updated': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
with open(STORAGE_FILE, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to save captures: {e}")
|
||||
|
||||
|
||||
# Load existing data on startup
|
||||
load_captures()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MIDDLEWARE
|
||||
# =============================================================================
|
||||
@@ -120,16 +164,101 @@ async def get_stats():
|
||||
# Count unique protocols as proxy for unique devices
|
||||
unique_protocols = len(set(c.get("protocol", "Unknown") for c in captures_storage))
|
||||
|
||||
# Count by data source
|
||||
data_sources = {}
|
||||
for capture in captures_storage:
|
||||
source = capture.get("data_source", "unknown")
|
||||
data_sources[source] = data_sources.get(source, 0) + 1
|
||||
|
||||
return {
|
||||
"total_captures": len(captures_storage),
|
||||
"unique_devices": unique_protocols,
|
||||
"coverage_area_km2": 0,
|
||||
"total_contributors": 1 if len(captures_storage) > 0 else 0,
|
||||
"frequency_distribution": freq_dist,
|
||||
"data_sources": data_sources,
|
||||
"captures_timeline": []
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/admin/cleanup")
|
||||
async def cleanup_test_data(data_source: str = "test"):
|
||||
"""
|
||||
Delete captures by data source (test, mock, or specific session)
|
||||
|
||||
Query params:
|
||||
- data_source: The data source to delete (test, mock, production)
|
||||
"""
|
||||
global captures_storage
|
||||
|
||||
initial_count = len(captures_storage)
|
||||
|
||||
# Filter out captures matching the data source
|
||||
captures_storage = [
|
||||
c for c in captures_storage
|
||||
if c.get("data_source", "production") != data_source
|
||||
]
|
||||
|
||||
deleted_count = initial_count - len(captures_storage)
|
||||
|
||||
# Save updated data
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Deleted {deleted_count} captures with data_source='{data_source}'",
|
||||
"deleted_count": deleted_count,
|
||||
"remaining_count": len(captures_storage)
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/admin/cleanup/session/{session_id}")
|
||||
async def cleanup_session(session_id: str):
|
||||
"""Delete all captures from a specific session"""
|
||||
global captures_storage
|
||||
|
||||
initial_count = len(captures_storage)
|
||||
|
||||
# Filter out captures matching the session
|
||||
captures_storage = [
|
||||
c for c in captures_storage
|
||||
if c.get("session_id", "") != session_id
|
||||
]
|
||||
|
||||
deleted_count = initial_count - len(captures_storage)
|
||||
|
||||
# Save updated data
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Deleted {deleted_count} captures from session '{session_id}'",
|
||||
"deleted_count": deleted_count,
|
||||
"remaining_count": len(captures_storage)
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/admin/cleanup/all")
|
||||
async def cleanup_all_data():
|
||||
"""Delete ALL captures (use with caution!)"""
|
||||
global captures_storage, upload_counter
|
||||
|
||||
deleted_count = len(captures_storage)
|
||||
|
||||
captures_storage = []
|
||||
upload_counter = 0
|
||||
|
||||
# Save empty data
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Deleted all {deleted_count} captures",
|
||||
"deleted_count": deleted_count,
|
||||
"remaining_count": 0
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/v1/captures/upload")
|
||||
async def upload_captures(
|
||||
files: List[UploadFile] = File(...),
|
||||
@@ -185,6 +314,8 @@ async def upload_captures(
|
||||
"longitude": gps_coords.longitude if gps_coords else manifest_data.get("captures", [{}])[0].get("longitude"),
|
||||
"gps_source": gps_coords.source if gps_coords else "manual",
|
||||
"timestamp": manifest_data.get("captures", [{}])[0].get("timestamp", ""),
|
||||
"data_source": manifest_data.get("data_source", "production"), # production, test, or mock
|
||||
"session_id": manifest_data.get("session_uuid", ""),
|
||||
}
|
||||
|
||||
# Store in memory
|
||||
@@ -201,6 +332,9 @@ async def upload_captures(
|
||||
if temp_path and os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
# Save to persistent storage
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Processed {len(successful)} files successfully",
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
"""
|
||||
Wardriving Data Importer
|
||||
|
||||
Imports GPS-tagged SubGhz captures from various formats:
|
||||
- Wigle-WiFi CSV format (adapted for SubGhz)
|
||||
- GPS coordinate JSON files
|
||||
- Batch .sub files with companion GPS data
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from dataclasses import dataclass
|
||||
from loguru import logger
|
||||
|
||||
from .sub_parser import SubFileParser
|
||||
from .gps_extractor import GPSFilenameExtractor, GPSCoordinates
|
||||
|
||||
|
||||
@dataclass
|
||||
class WardrivingCapture:
|
||||
"""Single wardriving capture with GPS and RF data"""
|
||||
filename: str
|
||||
frequency: int # Hz
|
||||
protocol: str
|
||||
latitude: float
|
||||
longitude: float
|
||||
altitude: Optional[float] = None
|
||||
accuracy: Optional[float] = None
|
||||
timestamp: Optional[str] = None
|
||||
rssi: Optional[int] = None
|
||||
session_id: Optional[str] = None
|
||||
gps_source: str = "wardriving"
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
"""Convert to dictionary for API"""
|
||||
return {
|
||||
'filename': self.filename,
|
||||
'frequency': self.frequency,
|
||||
'protocol': self.protocol,
|
||||
'latitude': self.latitude,
|
||||
'longitude': self.longitude,
|
||||
'altitude': self.altitude,
|
||||
'accuracy': self.accuracy,
|
||||
'timestamp': self.timestamp,
|
||||
'rssi': self.rssi,
|
||||
'session_id': self.session_id,
|
||||
'gps_source': self.gps_source
|
||||
}
|
||||
|
||||
|
||||
class WigleCSVImporter:
|
||||
"""
|
||||
Import SubGhz wardriving data from Wigle-WiFi CSV format
|
||||
|
||||
Format:
|
||||
WigleWifi-1.4,appRelease=SubGHz-Wardrive-1.0,...
|
||||
SignalHash,Frequency(MHz),Protocol,RSSI(dBm),Latitude,Longitude,Altitude,Accuracy,...
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.sub_parser = SubFileParser()
|
||||
self.captures: List[WardrivingCapture] = []
|
||||
|
||||
def parse_csv(self, csv_path: str) -> List[WardrivingCapture]:
|
||||
"""Parse Wigle CSV file"""
|
||||
path = Path(csv_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"CSV file not found: {csv_path}")
|
||||
|
||||
self.captures = []
|
||||
|
||||
with open(path, 'r') as f:
|
||||
# Read header line
|
||||
header_line = f.readline().strip()
|
||||
if not header_line.startswith('WigleWifi'):
|
||||
raise ValueError("Not a valid Wigle CSV file")
|
||||
|
||||
# Read column headers
|
||||
reader = csv.DictReader(f)
|
||||
|
||||
for row in reader:
|
||||
try:
|
||||
capture = self._parse_row(row, path.parent)
|
||||
if capture:
|
||||
self.captures.append(capture)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse row: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Parsed {len(self.captures)} captures from {csv_path}")
|
||||
return self.captures
|
||||
|
||||
def _parse_row(self, row: Dict, base_dir: Path) -> Optional[WardrivingCapture]:
|
||||
"""Parse single CSV row"""
|
||||
# Get frequency in Hz
|
||||
freq_mhz = float(row.get('Frequency(MHz)', 0))
|
||||
frequency = int(freq_mhz * 1e6)
|
||||
|
||||
# Get GPS coordinates
|
||||
lat = float(row.get('Latitude', 0))
|
||||
lon = float(row.get('Longitude', 0))
|
||||
|
||||
if lat == 0 or lon == 0:
|
||||
return None
|
||||
|
||||
# Get .sub filename
|
||||
filename = row.get('BruceFile', '')
|
||||
if not filename:
|
||||
filename = f"capture_{row.get('SignalHash', 'unknown')}.sub"
|
||||
|
||||
# Parse altitude and accuracy if available
|
||||
altitude = None
|
||||
if row.get('Altitude'):
|
||||
try:
|
||||
altitude = float(row['Altitude'])
|
||||
except:
|
||||
pass
|
||||
|
||||
accuracy = None
|
||||
if row.get('Accuracy'):
|
||||
try:
|
||||
accuracy = float(row['Accuracy'])
|
||||
except:
|
||||
pass
|
||||
|
||||
# Get RSSI
|
||||
rssi = None
|
||||
if row.get('RSSI(dBm)'):
|
||||
try:
|
||||
rssi = int(row['RSSI(dBm)'])
|
||||
except:
|
||||
pass
|
||||
|
||||
return WardrivingCapture(
|
||||
filename=filename,
|
||||
frequency=frequency,
|
||||
protocol=row.get('Protocol', 'Unknown'),
|
||||
latitude=lat,
|
||||
longitude=lon,
|
||||
altitude=altitude,
|
||||
accuracy=accuracy,
|
||||
timestamp=row.get('FirstSeen', datetime.now().isoformat()),
|
||||
rssi=rssi,
|
||||
session_id=row.get('SessionID'),
|
||||
gps_source='wigle_csv'
|
||||
)
|
||||
|
||||
|
||||
class GPSJSONImporter:
|
||||
"""
|
||||
Import GPS coordinates from JSON files
|
||||
|
||||
Format:
|
||||
{
|
||||
"type": "gps_coordinate",
|
||||
"data": {
|
||||
"latitude": 34.0522,
|
||||
"longitude": -118.2437,
|
||||
"accuracy": 5.0,
|
||||
"timestamp": "2026-01-09T21:26:51Z"
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def parse_json(json_path: str) -> Optional[GPSCoordinates]:
|
||||
"""Parse GPS JSON file"""
|
||||
path = Path(json_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"JSON file not found: {json_path}")
|
||||
|
||||
with open(path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Extract GPS data
|
||||
gps_data = data.get('data', {})
|
||||
|
||||
lat = gps_data.get('latitude')
|
||||
lon = gps_data.get('longitude')
|
||||
|
||||
if lat is None or lon is None:
|
||||
return None
|
||||
|
||||
return GPSCoordinates(
|
||||
latitude=float(lat),
|
||||
longitude=float(lon),
|
||||
source='gps_json',
|
||||
accuracy=gps_data.get('accuracy'),
|
||||
altitude=gps_data.get('altitude')
|
||||
)
|
||||
|
||||
|
||||
class BatchImporter:
|
||||
"""
|
||||
Batch import .sub files with companion GPS data
|
||||
|
||||
Supports:
|
||||
1. CSV manifest (Wigle format)
|
||||
2. Individual JSON GPS files (one per .sub file)
|
||||
3. GPS in filenames
|
||||
4. Single JSON with multiple GPS points
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.sub_parser = SubFileParser()
|
||||
self.gps_extractor = GPSFilenameExtractor()
|
||||
self.csv_importer = WigleCSVImporter()
|
||||
self.captures: List[WardrivingCapture] = []
|
||||
|
||||
def import_from_directory(
|
||||
self,
|
||||
directory: str,
|
||||
csv_manifest: Optional[str] = None
|
||||
) -> List[WardrivingCapture]:
|
||||
"""
|
||||
Import all .sub files from directory
|
||||
|
||||
Priority:
|
||||
1. CSV manifest (if provided)
|
||||
2. Companion JSON files (filename.json for filename.sub)
|
||||
3. GPS in filename
|
||||
4. Skip file if no GPS found
|
||||
"""
|
||||
dir_path = Path(directory)
|
||||
if not dir_path.exists():
|
||||
raise FileNotFoundError(f"Directory not found: {directory}")
|
||||
|
||||
self.captures = []
|
||||
|
||||
# Method 1: CSV manifest
|
||||
if csv_manifest:
|
||||
csv_captures = self.csv_importer.parse_csv(csv_manifest)
|
||||
self.captures.extend(csv_captures)
|
||||
logger.info(f"Imported {len(csv_captures)} captures from CSV manifest")
|
||||
return self.captures
|
||||
|
||||
# Method 2: Process individual .sub files
|
||||
sub_files = list(dir_path.glob('*.sub'))
|
||||
logger.info(f"Found {len(sub_files)} .sub files")
|
||||
|
||||
for sub_file in sub_files:
|
||||
capture = self._import_sub_file(sub_file)
|
||||
if capture:
|
||||
self.captures.append(capture)
|
||||
|
||||
logger.info(f"Successfully imported {len(self.captures)} captures")
|
||||
return self.captures
|
||||
|
||||
def _import_sub_file(self, sub_path: Path) -> Optional[WardrivingCapture]:
|
||||
"""Import single .sub file with GPS from various sources"""
|
||||
|
||||
# Try to get GPS coordinates
|
||||
gps_coords = None
|
||||
|
||||
# 1. Check for companion JSON file
|
||||
json_path = sub_path.with_suffix('.json')
|
||||
if json_path.exists():
|
||||
gps_coords = GPSJSONImporter.parse_json(str(json_path))
|
||||
if gps_coords:
|
||||
gps_coords.source = 'companion_json'
|
||||
|
||||
# 2. Extract from filename
|
||||
if not gps_coords:
|
||||
gps_coords = self.gps_extractor.extract(sub_path.name)
|
||||
|
||||
# Skip if no GPS found
|
||||
if not gps_coords:
|
||||
logger.debug(f"No GPS for {sub_path.name}, skipping")
|
||||
return None
|
||||
|
||||
# Parse .sub file for RF metadata
|
||||
try:
|
||||
metadata = self.sub_parser.parse(str(sub_path))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse {sub_path.name}: {e}")
|
||||
return None
|
||||
|
||||
return WardrivingCapture(
|
||||
filename=sub_path.name,
|
||||
frequency=metadata.frequency if hasattr(metadata, 'frequency') else 0,
|
||||
protocol=metadata.protocol if hasattr(metadata, 'protocol') else 'Unknown',
|
||||
latitude=gps_coords.latitude,
|
||||
longitude=gps_coords.longitude,
|
||||
altitude=gps_coords.altitude,
|
||||
accuracy=gps_coords.accuracy,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
gps_source=gps_coords.source
|
||||
)
|
||||
|
||||
def to_upload_manifest(self) -> Dict:
|
||||
"""Convert captures to upload manifest format"""
|
||||
return {
|
||||
"session_uuid": f"batch_import_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
|
||||
"captures": [c.to_dict() for c in self.captures]
|
||||
}
|
||||
|
||||
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Test Wigle CSV import
|
||||
print("=" * 60)
|
||||
print("Testing Wigle CSV Import")
|
||||
print("=" * 60)
|
||||
|
||||
csv_importer = WigleCSVImporter()
|
||||
csv_path = "signatures/t-embed-rf/test_export_wigle.csv"
|
||||
|
||||
if Path(csv_path).exists():
|
||||
captures = csv_importer.parse_csv(csv_path)
|
||||
print(f"\nImported {len(captures)} captures from CSV")
|
||||
|
||||
for capture in captures:
|
||||
print(f"\n File: {capture.filename}")
|
||||
print(f" Frequency: {capture.frequency / 1e6:.2f} MHz")
|
||||
print(f" GPS: {capture.latitude}, {capture.longitude}")
|
||||
print(f" Accuracy: {capture.accuracy}m")
|
||||
|
||||
# Test batch import from directory
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing Batch Import")
|
||||
print("=" * 60)
|
||||
|
||||
batch_importer = BatchImporter()
|
||||
captures = batch_importer.import_from_directory("signatures/t-embed-rf")
|
||||
|
||||
print(f"\nImported {len(captures)} total captures")
|
||||
print(f"\nGPS Sources:")
|
||||
for source in set(c.gps_source for c in captures):
|
||||
count = sum(1 for c in captures if c.gps_source == source)
|
||||
print(f" {source}: {count}")
|
||||
Reference in New Issue
Block a user