f5d92f1d36
Major Features: - Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS) - Data source tracking (test/mock/production) - Database cleanup tools for managing test data - JSON file persistence for simplified server - UI improvements for map controls Backend: - Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter - Added data_source and session_id tracking to captures - New API endpoints: DELETE /api/v1/admin/cleanup - Auto-save/load functionality for captures_simple.json - Updated stats endpoint to show data source breakdown CLI Tools: - scripts/import_wardriving_data.py - Batch import with --data-source flag - scripts/cleanup_database.py - Clean by source, session, or all - scripts/create_test_dataset.py - Generate GPS-tagged test data - scripts/test_wardriving_import.py - Test suite for importers Frontend: - Fixed map controls z-index and positioning issues - Moved controls to top-right to avoid zoom button overlap - Fixed Leaflet zoom controls rendering over header - Changed controls to position:fixed for persistent visibility - Export map object to window.map for proper invalidateSize Documentation: - docs/WARDRIVING_IMPORT.md - Complete import guide - docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation - docs/TEST_LOCATIONS.md - Test GPS coordinates reference - TEST_RESULTS_SUMMARY.md - Format testing results Test Data: - Created test_dataset_gps with 15 captures across 5 US cities - All test data marked with data_source="test" for easy cleanup Testing: - Verified Wigle CSV import (1 capture from West LA) - Verified GPS filename import (3 captures from UCLA area) - Verified GPS JSON companion files (15 captures, 5 cities) - Verified cleanup functionality (deleted 15 test captures) - Verified data persistence across server restarts 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
187 lines
4.7 KiB
Python
Executable File
187 lines
4.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Cleanup Database - Remove test/mock data
|
|
|
|
Remove test and mock captures from the database
|
|
"""
|
|
|
|
import sys
|
|
import argparse
|
|
import requests
|
|
from pathlib import Path
|
|
|
|
|
|
def cleanup_by_source(api_url, data_source):
|
|
"""Delete captures by data source"""
|
|
print("=" * 80)
|
|
print(f"Cleaning up captures with data_source='{data_source}'")
|
|
print("=" * 80)
|
|
|
|
endpoint = f"{api_url}/api/v1/admin/cleanup?data_source={data_source}"
|
|
|
|
try:
|
|
response = requests.delete(endpoint)
|
|
|
|
if response.status_code == 200:
|
|
result = response.json()
|
|
print(f"\n✅ {result['message']}")
|
|
print(f" Deleted: {result['deleted_count']}")
|
|
print(f" Remaining: {result['remaining_count']}")
|
|
return 0
|
|
else:
|
|
print(f"\n❌ HTTP {response.status_code}: {response.text}")
|
|
return 1
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return 1
|
|
|
|
|
|
def cleanup_session(api_url, session_id):
|
|
"""Delete captures by session ID"""
|
|
print("=" * 80)
|
|
print(f"Cleaning up session: {session_id}")
|
|
print("=" * 80)
|
|
|
|
endpoint = f"{api_url}/api/v1/admin/cleanup/session/{session_id}"
|
|
|
|
try:
|
|
response = requests.delete(endpoint)
|
|
|
|
if response.status_code == 200:
|
|
result = response.json()
|
|
print(f"\n✅ {result['message']}")
|
|
print(f" Deleted: {result['deleted_count']}")
|
|
print(f" Remaining: {result['remaining_count']}")
|
|
return 0
|
|
else:
|
|
print(f"\n❌ HTTP {response.status_code}: {response.text}")
|
|
return 1
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return 1
|
|
|
|
|
|
def cleanup_all(api_url, confirm=False):
|
|
"""Delete ALL captures"""
|
|
print("=" * 80)
|
|
print("⚠️ WARNING: Deleting ALL captures!")
|
|
print("=" * 80)
|
|
|
|
if not confirm:
|
|
print("\n❌ Confirmation required. Use --confirm to proceed.")
|
|
return 1
|
|
|
|
endpoint = f"{api_url}/api/v1/admin/cleanup/all"
|
|
|
|
try:
|
|
response = requests.delete(endpoint)
|
|
|
|
if response.status_code == 200:
|
|
result = response.json()
|
|
print(f"\n✅ {result['message']}")
|
|
print(f" Deleted: {result['deleted_count']}")
|
|
return 0
|
|
else:
|
|
print(f"\n❌ HTTP {response.status_code}: {response.text}")
|
|
return 1
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return 1
|
|
|
|
|
|
def show_stats(api_url):
|
|
"""Show current database stats"""
|
|
try:
|
|
response = requests.get(f"{api_url}/api/v1/stats/summary")
|
|
|
|
if response.status_code == 200:
|
|
stats = response.json()
|
|
print("\n📊 Current Database Stats:")
|
|
print(f" Total captures: {stats.get('total_captures', 0)}")
|
|
|
|
if 'data_sources' in stats:
|
|
print("\n By data source:")
|
|
for source, count in stats['data_sources'].items():
|
|
print(f" {source}: {count}")
|
|
|
|
print()
|
|
|
|
except Exception as e:
|
|
print(f"⚠️ Could not fetch stats: {e}")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Cleanup test/mock data from GigLez database"
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--api-url',
|
|
default='http://localhost:8000',
|
|
help='GigLez API URL (default: http://localhost:8000)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--source',
|
|
choices=['test', 'mock', 'production'],
|
|
help='Delete captures by data source'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--session',
|
|
help='Delete captures by session ID'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--all',
|
|
action='store_true',
|
|
help='Delete ALL captures (requires --confirm)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--confirm',
|
|
action='store_true',
|
|
help='Confirm deletion (required for --all)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--stats',
|
|
action='store_true',
|
|
help='Show database stats before and after cleanup'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Show initial stats
|
|
if args.stats:
|
|
print("BEFORE cleanup:")
|
|
show_stats(args.api_url)
|
|
|
|
# Perform cleanup
|
|
result = 0
|
|
|
|
if args.all:
|
|
result = cleanup_all(args.api_url, args.confirm)
|
|
elif args.source:
|
|
result = cleanup_by_source(args.api_url, args.source)
|
|
elif args.session:
|
|
result = cleanup_session(args.api_url, args.session)
|
|
else:
|
|
print("❌ Error: Must specify --source, --session, or --all")
|
|
parser.print_help()
|
|
return 1
|
|
|
|
# Show final stats
|
|
if args.stats and result == 0:
|
|
print("\nAFTER cleanup:")
|
|
show_stats(args.api_url)
|
|
|
|
return result
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|