Files
giglez/scripts/cleanup_database.py
Trilltechnician f5d92f1d36 feat: Add wardriving data import system and cleanup tools
Major Features:
- Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS)
- Data source tracking (test/mock/production)
- Database cleanup tools for managing test data
- JSON file persistence for simplified server
- UI improvements for map controls

Backend:
- Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter
- Added data_source and session_id tracking to captures
- New API endpoints: DELETE /api/v1/admin/cleanup
- Auto-save/load functionality for captures_simple.json
- Updated stats endpoint to show data source breakdown

CLI Tools:
- scripts/import_wardriving_data.py - Batch import with --data-source flag
- scripts/cleanup_database.py - Clean by source, session, or all
- scripts/create_test_dataset.py - Generate GPS-tagged test data
- scripts/test_wardriving_import.py - Test suite for importers

Frontend:
- Fixed map controls z-index and positioning issues
- Moved controls to top-right to avoid zoom button overlap
- Fixed Leaflet zoom controls rendering over header
- Changed controls to position:fixed for persistent visibility
- Export map object to window.map for proper invalidateSize

Documentation:
- docs/WARDRIVING_IMPORT.md - Complete import guide
- docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation
- docs/TEST_LOCATIONS.md - Test GPS coordinates reference
- TEST_RESULTS_SUMMARY.md - Format testing results

Test Data:
- Created test_dataset_gps with 15 captures across 5 US cities
- All test data marked with data_source="test" for easy cleanup

Testing:
- Verified Wigle CSV import (1 capture from West LA)
- Verified GPS filename import (3 captures from UCLA area)
- Verified GPS JSON companion files (15 captures, 5 cities)
- Verified cleanup functionality (deleted 15 test captures)
- Verified data persistence across server restarts

🤖 Generated with Claude Code

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-14 07:48:01 -08:00

187 lines
4.7 KiB
Python
Executable File

#!/usr/bin/env python3
"""
Cleanup Database - Remove test/mock data
Remove test and mock captures from the database
"""
import sys
import argparse
import requests
from pathlib import Path
def cleanup_by_source(api_url, data_source):
"""Delete captures by data source"""
print("=" * 80)
print(f"Cleaning up captures with data_source='{data_source}'")
print("=" * 80)
endpoint = f"{api_url}/api/v1/admin/cleanup?data_source={data_source}"
try:
response = requests.delete(endpoint)
if response.status_code == 200:
result = response.json()
print(f"\n{result['message']}")
print(f" Deleted: {result['deleted_count']}")
print(f" Remaining: {result['remaining_count']}")
return 0
else:
print(f"\n❌ HTTP {response.status_code}: {response.text}")
return 1
except Exception as e:
print(f"\n❌ Error: {e}")
return 1
def cleanup_session(api_url, session_id):
"""Delete captures by session ID"""
print("=" * 80)
print(f"Cleaning up session: {session_id}")
print("=" * 80)
endpoint = f"{api_url}/api/v1/admin/cleanup/session/{session_id}"
try:
response = requests.delete(endpoint)
if response.status_code == 200:
result = response.json()
print(f"\n{result['message']}")
print(f" Deleted: {result['deleted_count']}")
print(f" Remaining: {result['remaining_count']}")
return 0
else:
print(f"\n❌ HTTP {response.status_code}: {response.text}")
return 1
except Exception as e:
print(f"\n❌ Error: {e}")
return 1
def cleanup_all(api_url, confirm=False):
"""Delete ALL captures"""
print("=" * 80)
print("⚠️ WARNING: Deleting ALL captures!")
print("=" * 80)
if not confirm:
print("\n❌ Confirmation required. Use --confirm to proceed.")
return 1
endpoint = f"{api_url}/api/v1/admin/cleanup/all"
try:
response = requests.delete(endpoint)
if response.status_code == 200:
result = response.json()
print(f"\n{result['message']}")
print(f" Deleted: {result['deleted_count']}")
return 0
else:
print(f"\n❌ HTTP {response.status_code}: {response.text}")
return 1
except Exception as e:
print(f"\n❌ Error: {e}")
return 1
def show_stats(api_url):
"""Show current database stats"""
try:
response = requests.get(f"{api_url}/api/v1/stats/summary")
if response.status_code == 200:
stats = response.json()
print("\n📊 Current Database Stats:")
print(f" Total captures: {stats.get('total_captures', 0)}")
if 'data_sources' in stats:
print("\n By data source:")
for source, count in stats['data_sources'].items():
print(f" {source}: {count}")
print()
except Exception as e:
print(f"⚠️ Could not fetch stats: {e}")
def main():
parser = argparse.ArgumentParser(
description="Cleanup test/mock data from GigLez database"
)
parser.add_argument(
'--api-url',
default='http://localhost:8000',
help='GigLez API URL (default: http://localhost:8000)'
)
parser.add_argument(
'--source',
choices=['test', 'mock', 'production'],
help='Delete captures by data source'
)
parser.add_argument(
'--session',
help='Delete captures by session ID'
)
parser.add_argument(
'--all',
action='store_true',
help='Delete ALL captures (requires --confirm)'
)
parser.add_argument(
'--confirm',
action='store_true',
help='Confirm deletion (required for --all)'
)
parser.add_argument(
'--stats',
action='store_true',
help='Show database stats before and after cleanup'
)
args = parser.parse_args()
# Show initial stats
if args.stats:
print("BEFORE cleanup:")
show_stats(args.api_url)
# Perform cleanup
result = 0
if args.all:
result = cleanup_all(args.api_url, args.confirm)
elif args.source:
result = cleanup_by_source(args.api_url, args.source)
elif args.session:
result = cleanup_session(args.api_url, args.session)
else:
print("❌ Error: Must specify --source, --session, or --all")
parser.print_help()
return 1
# Show final stats
if args.stats and result == 0:
print("\nAFTER cleanup:")
show_stats(args.api_url)
return result
if __name__ == '__main__':
sys.exit(main())