feat: Add wardriving data import system and cleanup tools
Major Features: - Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS) - Data source tracking (test/mock/production) - Database cleanup tools for managing test data - JSON file persistence for simplified server - UI improvements for map controls Backend: - Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter - Added data_source and session_id tracking to captures - New API endpoints: DELETE /api/v1/admin/cleanup - Auto-save/load functionality for captures_simple.json - Updated stats endpoint to show data source breakdown CLI Tools: - scripts/import_wardriving_data.py - Batch import with --data-source flag - scripts/cleanup_database.py - Clean by source, session, or all - scripts/create_test_dataset.py - Generate GPS-tagged test data - scripts/test_wardriving_import.py - Test suite for importers Frontend: - Fixed map controls z-index and positioning issues - Moved controls to top-right to avoid zoom button overlap - Fixed Leaflet zoom controls rendering over header - Changed controls to position:fixed for persistent visibility - Export map object to window.map for proper invalidateSize Documentation: - docs/WARDRIVING_IMPORT.md - Complete import guide - docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation - docs/TEST_LOCATIONS.md - Test GPS coordinates reference - TEST_RESULTS_SUMMARY.md - Format testing results Test Data: - Created test_dataset_gps with 15 captures across 5 US cities - All test data marked with data_source="test" for easy cleanup Testing: - Verified Wigle CSV import (1 capture from West LA) - Verified GPS filename import (3 captures from UCLA area) - Verified GPS JSON companion files (15 captures, 5 cities) - Verified cleanup functionality (deleted 15 test captures) - Verified data persistence across server restarts 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Executable
+221
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import Wardriving Data to GigLez
|
||||
|
||||
Upload GPS-tagged SubGhz captures from various formats to the platform
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import requests
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.wardriving_importer import (
|
||||
WigleCSVImporter,
|
||||
BatchImporter
|
||||
)
|
||||
|
||||
|
||||
def upload_to_server(captures, api_url="http://localhost:8000", data_source="test"):
|
||||
"""Upload captures to GigLez server"""
|
||||
print(f"\n📤 Uploading {len(captures)} captures to {api_url}")
|
||||
print(f" Data source: {data_source}")
|
||||
|
||||
session_id = f"wardrive_import_{len(captures)}"
|
||||
|
||||
# Group captures and upload in batches (API expects files + manifest)
|
||||
# For now, we'll create individual uploads since we have the metadata
|
||||
|
||||
upload_endpoint = f"{api_url}/api/v1/captures/upload"
|
||||
successful = 0
|
||||
failed = 0
|
||||
|
||||
for i, capture in enumerate(captures, 1):
|
||||
print(f"\n[{i}/{len(captures)}] Uploading {capture.filename}...")
|
||||
|
||||
# Create manifest for this capture
|
||||
manifest = {
|
||||
"session_uuid": session_id,
|
||||
"data_source": data_source, # Mark as test/mock/production
|
||||
"captures": [{
|
||||
"filename": capture.filename,
|
||||
"latitude": capture.latitude,
|
||||
"longitude": capture.longitude,
|
||||
"accuracy": capture.accuracy or 5.0,
|
||||
"altitude": capture.altitude,
|
||||
"timestamp": capture.timestamp
|
||||
}]
|
||||
}
|
||||
|
||||
# For simplified server, we'll use the batch upload endpoint
|
||||
# In real usage, you'd upload the actual .sub file
|
||||
# For now, create a mock .sub file content
|
||||
|
||||
try:
|
||||
# Create mock .sub file content
|
||||
sub_content = f"""Filetype: Flipper SubGhz Key File
|
||||
Version: 1
|
||||
Frequency: {capture.frequency}
|
||||
Preset: FuriHalSubGhzPresetOok650Async
|
||||
Protocol: {capture.protocol}
|
||||
""".encode('utf-8')
|
||||
|
||||
files = [('files', (capture.filename, sub_content, 'application/octet-stream'))]
|
||||
data = {'manifest': json.dumps(manifest)}
|
||||
|
||||
response = requests.post(upload_endpoint, files=files, data=data)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
if result.get('success'):
|
||||
print(f" ✅ Success")
|
||||
successful += 1
|
||||
else:
|
||||
print(f" ❌ Failed: {result.get('message')}")
|
||||
failed += 1
|
||||
else:
|
||||
print(f" ❌ HTTP {response.status_code}")
|
||||
failed += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n" + "=" * 60)
|
||||
print(f"Upload Complete:")
|
||||
print(f" Successful: {successful}")
|
||||
print(f" Failed: {failed}")
|
||||
print(f" Total: {len(captures)}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def import_csv(csv_path, api_url, data_source="test", dry_run=False):
|
||||
"""Import from Wigle CSV format"""
|
||||
print("=" * 60)
|
||||
print("Importing from Wigle CSV")
|
||||
print("=" * 60)
|
||||
print(f"File: {csv_path}")
|
||||
|
||||
importer = WigleCSVImporter()
|
||||
captures = importer.parse_csv(csv_path)
|
||||
|
||||
print(f"\n✅ Parsed {len(captures)} captures")
|
||||
|
||||
# Show sample
|
||||
if captures:
|
||||
print(f"\nSample capture:")
|
||||
c = captures[0]
|
||||
print(f" Filename: {c.filename}")
|
||||
print(f" Frequency: {c.frequency / 1e6:.2f} MHz")
|
||||
print(f" GPS: {c.latitude}, {c.longitude}")
|
||||
print(f" Accuracy: {c.accuracy}m")
|
||||
|
||||
if dry_run:
|
||||
print("\n⚠️ Dry run - not uploading")
|
||||
return
|
||||
|
||||
upload_to_server(captures, api_url, data_source)
|
||||
|
||||
|
||||
def import_directory(directory, api_url, csv_manifest=None, data_source="test", dry_run=False):
|
||||
"""Import from directory of .sub files"""
|
||||
print("=" * 60)
|
||||
print("Importing from Directory")
|
||||
print("=" * 60)
|
||||
print(f"Directory: {directory}")
|
||||
if csv_manifest:
|
||||
print(f"CSV Manifest: {csv_manifest}")
|
||||
|
||||
importer = BatchImporter()
|
||||
captures = importer.import_from_directory(directory, csv_manifest)
|
||||
|
||||
print(f"\n✅ Found {len(captures)} captures with GPS")
|
||||
|
||||
# Show GPS source breakdown
|
||||
sources = {}
|
||||
for capture in captures:
|
||||
sources[capture.gps_source] = sources.get(capture.gps_source, 0) + 1
|
||||
|
||||
print("\nGPS Sources:")
|
||||
for source, count in sorted(sources.items()):
|
||||
print(f" {source}: {count}")
|
||||
|
||||
if dry_run:
|
||||
print("\n⚠️ Dry run - not uploading")
|
||||
return
|
||||
|
||||
upload_to_server(captures, api_url, data_source)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Import wardriving data to GigLez platform"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'source',
|
||||
help='CSV file or directory containing .sub files'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--csv',
|
||||
help='CSV manifest file (for directory imports)',
|
||||
default=None
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--api-url',
|
||||
default='http://localhost:8000',
|
||||
help='GigLez API URL (default: http://localhost:8000)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Parse files but do not upload'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--data-source',
|
||||
default='test',
|
||||
choices=['test', 'mock', 'production'],
|
||||
help='Mark data source (default: test). Use "production" for real wardriving data.'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
source_path = Path(args.source)
|
||||
|
||||
if not source_path.exists():
|
||||
print(f"❌ Error: {args.source} not found")
|
||||
return 1
|
||||
|
||||
try:
|
||||
if source_path.is_file() and source_path.suffix == '.csv':
|
||||
# Import from CSV
|
||||
import_csv(str(source_path), args.api_url, args.data_source, args.dry_run)
|
||||
|
||||
elif source_path.is_dir():
|
||||
# Import from directory
|
||||
import_directory(str(source_path), args.api_url, args.csv, args.data_source, args.dry_run)
|
||||
|
||||
else:
|
||||
print(f"❌ Error: {args.source} must be a .csv file or directory")
|
||||
return 1
|
||||
|
||||
print("\n✅ Import complete!")
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user