feat: Add wardriving data import system and cleanup tools
Major Features: - Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS) - Data source tracking (test/mock/production) - Database cleanup tools for managing test data - JSON file persistence for simplified server - UI improvements for map controls Backend: - Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter - Added data_source and session_id tracking to captures - New API endpoints: DELETE /api/v1/admin/cleanup - Auto-save/load functionality for captures_simple.json - Updated stats endpoint to show data source breakdown CLI Tools: - scripts/import_wardriving_data.py - Batch import with --data-source flag - scripts/cleanup_database.py - Clean by source, session, or all - scripts/create_test_dataset.py - Generate GPS-tagged test data - scripts/test_wardriving_import.py - Test suite for importers Frontend: - Fixed map controls z-index and positioning issues - Moved controls to top-right to avoid zoom button overlap - Fixed Leaflet zoom controls rendering over header - Changed controls to position:fixed for persistent visibility - Export map object to window.map for proper invalidateSize Documentation: - docs/WARDRIVING_IMPORT.md - Complete import guide - docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation - docs/TEST_LOCATIONS.md - Test GPS coordinates reference - TEST_RESULTS_SUMMARY.md - Format testing results Test Data: - Created test_dataset_gps with 15 captures across 5 US cities - All test data marked with data_source="test" for easy cleanup Testing: - Verified Wigle CSV import (1 capture from West LA) - Verified GPS filename import (3 captures from UCLA area) - Verified GPS JSON companion files (15 captures, 5 cities) - Verified cleanup functionality (deleted 15 test captures) - Verified data persistence across server restarts 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Executable
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cleanup Database - Remove test/mock data
|
||||
|
||||
Remove test and mock captures from the database
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def cleanup_by_source(api_url, data_source):
|
||||
"""Delete captures by data source"""
|
||||
print("=" * 80)
|
||||
print(f"Cleaning up captures with data_source='{data_source}'")
|
||||
print("=" * 80)
|
||||
|
||||
endpoint = f"{api_url}/api/v1/admin/cleanup?data_source={data_source}"
|
||||
|
||||
try:
|
||||
response = requests.delete(endpoint)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"\n✅ {result['message']}")
|
||||
print(f" Deleted: {result['deleted_count']}")
|
||||
print(f" Remaining: {result['remaining_count']}")
|
||||
return 0
|
||||
else:
|
||||
print(f"\n❌ HTTP {response.status_code}: {response.text}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
return 1
|
||||
|
||||
|
||||
def cleanup_session(api_url, session_id):
|
||||
"""Delete captures by session ID"""
|
||||
print("=" * 80)
|
||||
print(f"Cleaning up session: {session_id}")
|
||||
print("=" * 80)
|
||||
|
||||
endpoint = f"{api_url}/api/v1/admin/cleanup/session/{session_id}"
|
||||
|
||||
try:
|
||||
response = requests.delete(endpoint)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"\n✅ {result['message']}")
|
||||
print(f" Deleted: {result['deleted_count']}")
|
||||
print(f" Remaining: {result['remaining_count']}")
|
||||
return 0
|
||||
else:
|
||||
print(f"\n❌ HTTP {response.status_code}: {response.text}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
return 1
|
||||
|
||||
|
||||
def cleanup_all(api_url, confirm=False):
|
||||
"""Delete ALL captures"""
|
||||
print("=" * 80)
|
||||
print("⚠️ WARNING: Deleting ALL captures!")
|
||||
print("=" * 80)
|
||||
|
||||
if not confirm:
|
||||
print("\n❌ Confirmation required. Use --confirm to proceed.")
|
||||
return 1
|
||||
|
||||
endpoint = f"{api_url}/api/v1/admin/cleanup/all"
|
||||
|
||||
try:
|
||||
response = requests.delete(endpoint)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"\n✅ {result['message']}")
|
||||
print(f" Deleted: {result['deleted_count']}")
|
||||
return 0
|
||||
else:
|
||||
print(f"\n❌ HTTP {response.status_code}: {response.text}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
return 1
|
||||
|
||||
|
||||
def show_stats(api_url):
|
||||
"""Show current database stats"""
|
||||
try:
|
||||
response = requests.get(f"{api_url}/api/v1/stats/summary")
|
||||
|
||||
if response.status_code == 200:
|
||||
stats = response.json()
|
||||
print("\n📊 Current Database Stats:")
|
||||
print(f" Total captures: {stats.get('total_captures', 0)}")
|
||||
|
||||
if 'data_sources' in stats:
|
||||
print("\n By data source:")
|
||||
for source, count in stats['data_sources'].items():
|
||||
print(f" {source}: {count}")
|
||||
|
||||
print()
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not fetch stats: {e}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Cleanup test/mock data from GigLez database"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--api-url',
|
||||
default='http://localhost:8000',
|
||||
help='GigLez API URL (default: http://localhost:8000)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--source',
|
||||
choices=['test', 'mock', 'production'],
|
||||
help='Delete captures by data source'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--session',
|
||||
help='Delete captures by session ID'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--all',
|
||||
action='store_true',
|
||||
help='Delete ALL captures (requires --confirm)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--confirm',
|
||||
action='store_true',
|
||||
help='Confirm deletion (required for --all)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--stats',
|
||||
action='store_true',
|
||||
help='Show database stats before and after cleanup'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Show initial stats
|
||||
if args.stats:
|
||||
print("BEFORE cleanup:")
|
||||
show_stats(args.api_url)
|
||||
|
||||
# Perform cleanup
|
||||
result = 0
|
||||
|
||||
if args.all:
|
||||
result = cleanup_all(args.api_url, args.confirm)
|
||||
elif args.source:
|
||||
result = cleanup_by_source(args.api_url, args.source)
|
||||
elif args.session:
|
||||
result = cleanup_session(args.api_url, args.session)
|
||||
else:
|
||||
print("❌ Error: Must specify --source, --session, or --all")
|
||||
parser.print_help()
|
||||
return 1
|
||||
|
||||
# Show final stats
|
||||
if args.stats and result == 0:
|
||||
print("\nAFTER cleanup:")
|
||||
show_stats(args.api_url)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Create Test Dataset with GPS Coordinates
|
||||
|
||||
Adds GPS coordinates to existing .sub files from Flipper Zero for testing
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
# Test GPS coordinates across different US cities
|
||||
TEST_LOCATIONS = {
|
||||
"los_angeles": {
|
||||
"name": "Los Angeles, CA",
|
||||
"coords": [(34.0522, -118.2437), (34.0736, -118.3596), (34.0478, -118.2348)]
|
||||
},
|
||||
"san_francisco": {
|
||||
"name": "San Francisco, CA",
|
||||
"coords": [(37.7749, -122.4194), (37.8044, -122.2712), (37.7558, -122.4449)]
|
||||
},
|
||||
"new_york": {
|
||||
"name": "New York, NY",
|
||||
"coords": [(40.7128, -74.0060), (40.7580, -73.9855), (40.6782, -73.9442)]
|
||||
},
|
||||
"chicago": {
|
||||
"name": "Chicago, IL",
|
||||
"coords": [(41.8781, -87.6298), (41.8919, -87.6051), (41.8369, -87.6847)]
|
||||
},
|
||||
"austin": {
|
||||
"name": "Austin, TX",
|
||||
"coords": [(30.2672, -97.7431), (30.3072, -97.7559), (30.2500, -97.7500)]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def create_gps_tagged_captures():
|
||||
"""Copy Flipper Zero captures and add GPS tags via filenames and JSON"""
|
||||
|
||||
# Source directory
|
||||
flipper_dir = Path("signatures/flipperzero-firmware/applications/debug/unit_tests/resources/unit_tests/subghz")
|
||||
|
||||
# Output directory
|
||||
output_dir = Path("signatures/test_dataset_gps")
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
print("=" * 80)
|
||||
print("Creating Test Dataset with GPS Coordinates")
|
||||
print("=" * 80)
|
||||
|
||||
if not flipper_dir.exists():
|
||||
print(f"❌ Flipper directory not found: {flipper_dir}")
|
||||
return
|
||||
|
||||
# Get all .sub files
|
||||
sub_files = list(flipper_dir.glob("*.sub"))
|
||||
|
||||
print(f"\n📁 Found {len(sub_files)} .sub files")
|
||||
print(f"📍 Creating GPS-tagged versions across {len(TEST_LOCATIONS)} cities")
|
||||
|
||||
created_count = 0
|
||||
location_idx = 0
|
||||
|
||||
# Create captures for each location
|
||||
for city_key, city_data in TEST_LOCATIONS.items():
|
||||
city_name = city_data["name"]
|
||||
coords_list = city_data["coords"]
|
||||
|
||||
print(f"\n🌎 {city_name}")
|
||||
|
||||
for i, (lat, lon) in enumerate(coords_list):
|
||||
if location_idx >= len(sub_files):
|
||||
break
|
||||
|
||||
source_file = sub_files[location_idx]
|
||||
|
||||
# Create GPS-tagged filename
|
||||
lat_str = f"{abs(lat):.4f}{'N' if lat >= 0 else 'S'}"
|
||||
lon_str = f"{abs(lon):.4f}{'W' if lon < 0 else 'E'}"
|
||||
|
||||
base_name = source_file.stem
|
||||
new_filename = f"{lat_str}_{lon_str}_{base_name}.sub"
|
||||
dest_file = output_dir / new_filename
|
||||
|
||||
# Copy .sub file
|
||||
shutil.copy(source_file, dest_file)
|
||||
|
||||
# Create companion GPS JSON
|
||||
gps_json = {
|
||||
"type": "gps_coordinate",
|
||||
"source": "test_dataset",
|
||||
"city": city_name,
|
||||
"data": {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"accuracy": 5.0 + (i * 2), # Vary accuracy
|
||||
"altitude": 10.0 + (i * 5),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
}
|
||||
|
||||
json_file = output_dir / new_filename.replace('.sub', '.json')
|
||||
with open(json_file, 'w') as f:
|
||||
json.dump(gps_json, f, indent=2)
|
||||
|
||||
print(f" ✅ {new_filename} → {lat:.4f}, {lon:.4f}")
|
||||
|
||||
created_count += 1
|
||||
location_idx += 1
|
||||
|
||||
# Create Wigle CSV manifest
|
||||
create_wigle_csv(output_dir)
|
||||
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"✅ Created {created_count} GPS-tagged captures")
|
||||
print(f"📂 Output directory: {output_dir}")
|
||||
print(f"{'=' * 80}")
|
||||
|
||||
return output_dir
|
||||
|
||||
|
||||
def create_wigle_csv(output_dir):
|
||||
"""Create Wigle CSV manifest for all captures"""
|
||||
|
||||
csv_file = output_dir / "test_wigle_manifest.csv"
|
||||
|
||||
# Get all .sub files
|
||||
sub_files = sorted(output_dir.glob("*.sub"))
|
||||
|
||||
with open(csv_file, 'w') as f:
|
||||
# Header
|
||||
f.write("WigleWifi-1.4,appRelease=GigLez-Test-1.0,model=TestDataset\n")
|
||||
f.write("SignalHash,Frequency(MHz),Protocol,RSSI(dBm),Latitude,Longitude,Altitude,Accuracy,FirstSeen,LastSeen,CaptureCount,Interpolated,Confidence,BruceFile,SessionID\n")
|
||||
|
||||
# Parse each file
|
||||
for sub_file in sub_files:
|
||||
# Parse .sub file for frequency/protocol
|
||||
frequency = 433.92 # Default
|
||||
protocol = "RAW"
|
||||
|
||||
with open(sub_file, 'r') as sf:
|
||||
for line in sf:
|
||||
if line.startswith('Frequency:'):
|
||||
freq_hz = int(line.split(':')[1].strip())
|
||||
frequency = freq_hz / 1e6
|
||||
elif line.startswith('Protocol:'):
|
||||
protocol = line.split(':')[1].strip()
|
||||
|
||||
# Extract GPS from filename
|
||||
filename = sub_file.name
|
||||
parts = filename.split('_')
|
||||
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
# Parse latitude
|
||||
lat_str = parts[0]
|
||||
lat_val = float(lat_str[:-1])
|
||||
if lat_str[-1] == 'S':
|
||||
lat_val = -lat_val
|
||||
|
||||
# Parse longitude
|
||||
lon_str = parts[1]
|
||||
lon_val = float(lon_str[:-1])
|
||||
if lon_str[-1] == 'W':
|
||||
lon_val = -lon_val
|
||||
|
||||
# Write CSV row
|
||||
hash_val = f"test_{sub_file.stem}"
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
f.write(f"{hash_val},{frequency:.3f},{protocol},-70,{lat_val},{lon_val},10.0,5.0,{timestamp},{timestamp},1,false,1.0,{filename},test_session\n")
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to parse GPS from {filename}: {e}")
|
||||
|
||||
print(f"\n📄 Created Wigle CSV: {csv_file.name} ({len(sub_files)} captures)")
|
||||
|
||||
|
||||
def main():
|
||||
"""Create test dataset"""
|
||||
|
||||
try:
|
||||
output_dir = create_gps_tagged_captures()
|
||||
|
||||
print("\n📤 To upload all test data:")
|
||||
print(f" python3 scripts/import_wardriving_data.py {output_dir}/")
|
||||
print("\n OR")
|
||||
print(f" python3 scripts/import_wardriving_data.py {output_dir}/test_wigle_manifest.csv")
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+221
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import Wardriving Data to GigLez
|
||||
|
||||
Upload GPS-tagged SubGhz captures from various formats to the platform
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import requests
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.wardriving_importer import (
|
||||
WigleCSVImporter,
|
||||
BatchImporter
|
||||
)
|
||||
|
||||
|
||||
def upload_to_server(captures, api_url="http://localhost:8000", data_source="test"):
|
||||
"""Upload captures to GigLez server"""
|
||||
print(f"\n📤 Uploading {len(captures)} captures to {api_url}")
|
||||
print(f" Data source: {data_source}")
|
||||
|
||||
session_id = f"wardrive_import_{len(captures)}"
|
||||
|
||||
# Group captures and upload in batches (API expects files + manifest)
|
||||
# For now, we'll create individual uploads since we have the metadata
|
||||
|
||||
upload_endpoint = f"{api_url}/api/v1/captures/upload"
|
||||
successful = 0
|
||||
failed = 0
|
||||
|
||||
for i, capture in enumerate(captures, 1):
|
||||
print(f"\n[{i}/{len(captures)}] Uploading {capture.filename}...")
|
||||
|
||||
# Create manifest for this capture
|
||||
manifest = {
|
||||
"session_uuid": session_id,
|
||||
"data_source": data_source, # Mark as test/mock/production
|
||||
"captures": [{
|
||||
"filename": capture.filename,
|
||||
"latitude": capture.latitude,
|
||||
"longitude": capture.longitude,
|
||||
"accuracy": capture.accuracy or 5.0,
|
||||
"altitude": capture.altitude,
|
||||
"timestamp": capture.timestamp
|
||||
}]
|
||||
}
|
||||
|
||||
# For simplified server, we'll use the batch upload endpoint
|
||||
# In real usage, you'd upload the actual .sub file
|
||||
# For now, create a mock .sub file content
|
||||
|
||||
try:
|
||||
# Create mock .sub file content
|
||||
sub_content = f"""Filetype: Flipper SubGhz Key File
|
||||
Version: 1
|
||||
Frequency: {capture.frequency}
|
||||
Preset: FuriHalSubGhzPresetOok650Async
|
||||
Protocol: {capture.protocol}
|
||||
""".encode('utf-8')
|
||||
|
||||
files = [('files', (capture.filename, sub_content, 'application/octet-stream'))]
|
||||
data = {'manifest': json.dumps(manifest)}
|
||||
|
||||
response = requests.post(upload_endpoint, files=files, data=data)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
if result.get('success'):
|
||||
print(f" ✅ Success")
|
||||
successful += 1
|
||||
else:
|
||||
print(f" ❌ Failed: {result.get('message')}")
|
||||
failed += 1
|
||||
else:
|
||||
print(f" ❌ HTTP {response.status_code}")
|
||||
failed += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n" + "=" * 60)
|
||||
print(f"Upload Complete:")
|
||||
print(f" Successful: {successful}")
|
||||
print(f" Failed: {failed}")
|
||||
print(f" Total: {len(captures)}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def import_csv(csv_path, api_url, data_source="test", dry_run=False):
|
||||
"""Import from Wigle CSV format"""
|
||||
print("=" * 60)
|
||||
print("Importing from Wigle CSV")
|
||||
print("=" * 60)
|
||||
print(f"File: {csv_path}")
|
||||
|
||||
importer = WigleCSVImporter()
|
||||
captures = importer.parse_csv(csv_path)
|
||||
|
||||
print(f"\n✅ Parsed {len(captures)} captures")
|
||||
|
||||
# Show sample
|
||||
if captures:
|
||||
print(f"\nSample capture:")
|
||||
c = captures[0]
|
||||
print(f" Filename: {c.filename}")
|
||||
print(f" Frequency: {c.frequency / 1e6:.2f} MHz")
|
||||
print(f" GPS: {c.latitude}, {c.longitude}")
|
||||
print(f" Accuracy: {c.accuracy}m")
|
||||
|
||||
if dry_run:
|
||||
print("\n⚠️ Dry run - not uploading")
|
||||
return
|
||||
|
||||
upload_to_server(captures, api_url, data_source)
|
||||
|
||||
|
||||
def import_directory(directory, api_url, csv_manifest=None, data_source="test", dry_run=False):
|
||||
"""Import from directory of .sub files"""
|
||||
print("=" * 60)
|
||||
print("Importing from Directory")
|
||||
print("=" * 60)
|
||||
print(f"Directory: {directory}")
|
||||
if csv_manifest:
|
||||
print(f"CSV Manifest: {csv_manifest}")
|
||||
|
||||
importer = BatchImporter()
|
||||
captures = importer.import_from_directory(directory, csv_manifest)
|
||||
|
||||
print(f"\n✅ Found {len(captures)} captures with GPS")
|
||||
|
||||
# Show GPS source breakdown
|
||||
sources = {}
|
||||
for capture in captures:
|
||||
sources[capture.gps_source] = sources.get(capture.gps_source, 0) + 1
|
||||
|
||||
print("\nGPS Sources:")
|
||||
for source, count in sorted(sources.items()):
|
||||
print(f" {source}: {count}")
|
||||
|
||||
if dry_run:
|
||||
print("\n⚠️ Dry run - not uploading")
|
||||
return
|
||||
|
||||
upload_to_server(captures, api_url, data_source)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Import wardriving data to GigLez platform"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'source',
|
||||
help='CSV file or directory containing .sub files'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--csv',
|
||||
help='CSV manifest file (for directory imports)',
|
||||
default=None
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--api-url',
|
||||
default='http://localhost:8000',
|
||||
help='GigLez API URL (default: http://localhost:8000)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Parse files but do not upload'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--data-source',
|
||||
default='test',
|
||||
choices=['test', 'mock', 'production'],
|
||||
help='Mark data source (default: test). Use "production" for real wardriving data.'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
source_path = Path(args.source)
|
||||
|
||||
if not source_path.exists():
|
||||
print(f"❌ Error: {args.source} not found")
|
||||
return 1
|
||||
|
||||
try:
|
||||
if source_path.is_file() and source_path.suffix == '.csv':
|
||||
# Import from CSV
|
||||
import_csv(str(source_path), args.api_url, args.data_source, args.dry_run)
|
||||
|
||||
elif source_path.is_dir():
|
||||
# Import from directory
|
||||
import_directory(str(source_path), args.api_url, args.csv, args.data_source, args.dry_run)
|
||||
|
||||
else:
|
||||
print(f"❌ Error: {args.source} must be a .csv file or directory")
|
||||
return 1
|
||||
|
||||
print("\n✅ Import complete!")
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Wardriving Data Import
|
||||
|
||||
Tests importing GPS-tagged SubGhz captures from various formats
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.wardriving_importer import (
|
||||
WigleCSVImporter,
|
||||
GPSJSONImporter,
|
||||
BatchImporter
|
||||
)
|
||||
|
||||
|
||||
def test_wigle_csv_import():
|
||||
"""Test Wigle CSV format import"""
|
||||
print("=" * 80)
|
||||
print("TEST 1: Wigle CSV Import")
|
||||
print("=" * 80)
|
||||
|
||||
csv_path = "signatures/t-embed-rf/test_export_wigle.csv"
|
||||
|
||||
if not Path(csv_path).exists():
|
||||
print(f"❌ CSV file not found: {csv_path}")
|
||||
return
|
||||
|
||||
importer = WigleCSVImporter()
|
||||
captures = importer.parse_csv(csv_path)
|
||||
|
||||
print(f"\n✅ Imported {len(captures)} captures from CSV")
|
||||
|
||||
for i, capture in enumerate(captures, 1):
|
||||
print(f"\nCapture {i}:")
|
||||
print(f" Filename: {capture.filename}")
|
||||
print(f" Frequency: {capture.frequency / 1e6:.2f} MHz")
|
||||
print(f" Protocol: {capture.protocol}")
|
||||
print(f" GPS: {capture.latitude}, {capture.longitude}")
|
||||
print(f" Accuracy: {capture.accuracy}m")
|
||||
print(f" Timestamp: {capture.timestamp}")
|
||||
print(f" GPS Source: {capture.gps_source}")
|
||||
|
||||
|
||||
def test_gps_json_import():
|
||||
"""Test GPS JSON format import"""
|
||||
print("\n" + "=" * 80)
|
||||
print("TEST 2: GPS JSON Import")
|
||||
print("=" * 80)
|
||||
|
||||
json_files = list(Path("signatures/t-embed-rf").glob("gps_coordinates_*.json"))
|
||||
|
||||
if not json_files:
|
||||
print("❌ No GPS JSON files found")
|
||||
return
|
||||
|
||||
print(f"\nFound {len(json_files)} GPS JSON files")
|
||||
|
||||
for json_file in json_files[:3]: # Test first 3
|
||||
coords = GPSJSONImporter.parse_json(str(json_file))
|
||||
|
||||
if coords:
|
||||
print(f"\n✅ {json_file.name}")
|
||||
print(f" Latitude: {coords.latitude}")
|
||||
print(f" Longitude: {coords.longitude}")
|
||||
print(f" Accuracy: {coords.accuracy}m")
|
||||
print(f" Source: {coords.source}")
|
||||
else:
|
||||
print(f"\n❌ {json_file.name} - No GPS data")
|
||||
|
||||
|
||||
def test_batch_import():
|
||||
"""Test batch directory import"""
|
||||
print("\n" + "=" * 80)
|
||||
print("TEST 3: Batch Directory Import")
|
||||
print("=" * 80)
|
||||
|
||||
importer = BatchImporter()
|
||||
captures = importer.import_from_directory("signatures/t-embed-rf")
|
||||
|
||||
print(f"\n✅ Imported {len(captures)} total captures")
|
||||
|
||||
# Group by GPS source
|
||||
print("\nGPS Sources:")
|
||||
sources = {}
|
||||
for capture in captures:
|
||||
sources[capture.gps_source] = sources.get(capture.gps_source, 0) + 1
|
||||
|
||||
for source, count in sorted(sources.items()):
|
||||
print(f" {source}: {count}")
|
||||
|
||||
# Show sample captures
|
||||
print("\nSample Captures:")
|
||||
for i, capture in enumerate(captures[:5], 1):
|
||||
print(f"\n {i}. {capture.filename}")
|
||||
print(f" Frequency: {capture.frequency / 1e6:.2f} MHz")
|
||||
print(f" GPS: {capture.latitude}, {capture.longitude}")
|
||||
print(f" Source: {capture.gps_source}")
|
||||
|
||||
|
||||
def test_upload_manifest():
|
||||
"""Test generating upload manifest"""
|
||||
print("\n" + "=" * 80)
|
||||
print("TEST 4: Upload Manifest Generation")
|
||||
print("=" * 80)
|
||||
|
||||
importer = BatchImporter()
|
||||
captures = importer.import_from_directory("signatures/t-embed-rf")
|
||||
|
||||
manifest = importer.to_upload_manifest()
|
||||
|
||||
print(f"\n✅ Generated manifest with {len(manifest['captures'])} captures")
|
||||
print(f"Session UUID: {manifest['session_uuid']}")
|
||||
|
||||
print("\nSample capture data:")
|
||||
if manifest['captures']:
|
||||
import json
|
||||
print(json.dumps(manifest['captures'][0], indent=2))
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all tests"""
|
||||
print("\n🧪 WARDRIVING DATA IMPORT TESTS")
|
||||
print("=" * 80)
|
||||
|
||||
try:
|
||||
test_wigle_csv_import()
|
||||
test_gps_json_import()
|
||||
test_batch_import()
|
||||
test_upload_manifest()
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("✅ ALL TESTS COMPLETE")
|
||||
print("=" * 80)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user