feat: Add wardriving data import system and cleanup tools
Major Features: - Multi-format GPS data import (Wigle CSV, GPS JSON, filename GPS) - Data source tracking (test/mock/production) - Database cleanup tools for managing test data - JSON file persistence for simplified server - UI improvements for map controls Backend: - Added wardriving_importer.py with WigleCSVImporter, GPSJSONImporter, BatchImporter - Added data_source and session_id tracking to captures - New API endpoints: DELETE /api/v1/admin/cleanup - Auto-save/load functionality for captures_simple.json - Updated stats endpoint to show data source breakdown CLI Tools: - scripts/import_wardriving_data.py - Batch import with --data-source flag - scripts/cleanup_database.py - Clean by source, session, or all - scripts/create_test_dataset.py - Generate GPS-tagged test data - scripts/test_wardriving_import.py - Test suite for importers Frontend: - Fixed map controls z-index and positioning issues - Moved controls to top-right to avoid zoom button overlap - Fixed Leaflet zoom controls rendering over header - Changed controls to position:fixed for persistent visibility - Export map object to window.map for proper invalidateSize Documentation: - docs/WARDRIVING_IMPORT.md - Complete import guide - docs/DATA_CLEANUP_GUIDE.md - Cleanup system documentation - docs/TEST_LOCATIONS.md - Test GPS coordinates reference - TEST_RESULTS_SUMMARY.md - Format testing results Test Data: - Created test_dataset_gps with 15 captures across 5 US cities - All test data marked with data_source="test" for easy cleanup Testing: - Verified Wigle CSV import (1 capture from West LA) - Verified GPS filename import (3 captures from UCLA area) - Verified GPS JSON companion files (15 captures, 5 cities) - Verified cleanup functionality (deleted 15 test captures) - Verified data persistence across server restarts 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from datetime import datetime
|
||||
from fastapi import FastAPI, Request, UploadFile, File, Form
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
@@ -38,11 +39,54 @@ BASE_DIR = Path(__file__).parent.parent.parent
|
||||
app.mount("/static", StaticFiles(directory=str(BASE_DIR / "static")), name="static")
|
||||
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
|
||||
|
||||
# Persistent storage file
|
||||
STORAGE_FILE = BASE_DIR / "data" / "captures_simple.json"
|
||||
STORAGE_FILE.parent.mkdir(exist_ok=True)
|
||||
|
||||
# In-memory storage for uploaded captures (simplified mode)
|
||||
captures_storage = []
|
||||
upload_counter = 0
|
||||
|
||||
|
||||
def load_captures():
|
||||
"""Load captures from JSON file"""
|
||||
global captures_storage, upload_counter
|
||||
|
||||
if STORAGE_FILE.exists():
|
||||
try:
|
||||
with open(STORAGE_FILE, 'r') as f:
|
||||
data = json.load(f)
|
||||
captures_storage = data.get('captures', [])
|
||||
upload_counter = data.get('upload_counter', 0)
|
||||
print(f"✅ Loaded {len(captures_storage)} captures from {STORAGE_FILE}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to load captures: {e}")
|
||||
captures_storage = []
|
||||
upload_counter = 0
|
||||
else:
|
||||
print(f"ℹ️ No existing captures file at {STORAGE_FILE}")
|
||||
|
||||
|
||||
def save_captures():
|
||||
"""Save captures to JSON file"""
|
||||
try:
|
||||
data = {
|
||||
'captures': captures_storage,
|
||||
'upload_counter': upload_counter,
|
||||
'last_updated': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
with open(STORAGE_FILE, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to save captures: {e}")
|
||||
|
||||
|
||||
# Load existing data on startup
|
||||
load_captures()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MIDDLEWARE
|
||||
# =============================================================================
|
||||
@@ -120,16 +164,101 @@ async def get_stats():
|
||||
# Count unique protocols as proxy for unique devices
|
||||
unique_protocols = len(set(c.get("protocol", "Unknown") for c in captures_storage))
|
||||
|
||||
# Count by data source
|
||||
data_sources = {}
|
||||
for capture in captures_storage:
|
||||
source = capture.get("data_source", "unknown")
|
||||
data_sources[source] = data_sources.get(source, 0) + 1
|
||||
|
||||
return {
|
||||
"total_captures": len(captures_storage),
|
||||
"unique_devices": unique_protocols,
|
||||
"coverage_area_km2": 0,
|
||||
"total_contributors": 1 if len(captures_storage) > 0 else 0,
|
||||
"frequency_distribution": freq_dist,
|
||||
"data_sources": data_sources,
|
||||
"captures_timeline": []
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/admin/cleanup")
|
||||
async def cleanup_test_data(data_source: str = "test"):
|
||||
"""
|
||||
Delete captures by data source (test, mock, or specific session)
|
||||
|
||||
Query params:
|
||||
- data_source: The data source to delete (test, mock, production)
|
||||
"""
|
||||
global captures_storage
|
||||
|
||||
initial_count = len(captures_storage)
|
||||
|
||||
# Filter out captures matching the data source
|
||||
captures_storage = [
|
||||
c for c in captures_storage
|
||||
if c.get("data_source", "production") != data_source
|
||||
]
|
||||
|
||||
deleted_count = initial_count - len(captures_storage)
|
||||
|
||||
# Save updated data
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Deleted {deleted_count} captures with data_source='{data_source}'",
|
||||
"deleted_count": deleted_count,
|
||||
"remaining_count": len(captures_storage)
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/admin/cleanup/session/{session_id}")
|
||||
async def cleanup_session(session_id: str):
|
||||
"""Delete all captures from a specific session"""
|
||||
global captures_storage
|
||||
|
||||
initial_count = len(captures_storage)
|
||||
|
||||
# Filter out captures matching the session
|
||||
captures_storage = [
|
||||
c for c in captures_storage
|
||||
if c.get("session_id", "") != session_id
|
||||
]
|
||||
|
||||
deleted_count = initial_count - len(captures_storage)
|
||||
|
||||
# Save updated data
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Deleted {deleted_count} captures from session '{session_id}'",
|
||||
"deleted_count": deleted_count,
|
||||
"remaining_count": len(captures_storage)
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/admin/cleanup/all")
|
||||
async def cleanup_all_data():
|
||||
"""Delete ALL captures (use with caution!)"""
|
||||
global captures_storage, upload_counter
|
||||
|
||||
deleted_count = len(captures_storage)
|
||||
|
||||
captures_storage = []
|
||||
upload_counter = 0
|
||||
|
||||
# Save empty data
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Deleted all {deleted_count} captures",
|
||||
"deleted_count": deleted_count,
|
||||
"remaining_count": 0
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/v1/captures/upload")
|
||||
async def upload_captures(
|
||||
files: List[UploadFile] = File(...),
|
||||
@@ -185,6 +314,8 @@ async def upload_captures(
|
||||
"longitude": gps_coords.longitude if gps_coords else manifest_data.get("captures", [{}])[0].get("longitude"),
|
||||
"gps_source": gps_coords.source if gps_coords else "manual",
|
||||
"timestamp": manifest_data.get("captures", [{}])[0].get("timestamp", ""),
|
||||
"data_source": manifest_data.get("data_source", "production"), # production, test, or mock
|
||||
"session_id": manifest_data.get("session_uuid", ""),
|
||||
}
|
||||
|
||||
# Store in memory
|
||||
@@ -201,6 +332,9 @@ async def upload_captures(
|
||||
if temp_path and os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
# Save to persistent storage
|
||||
save_captures()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Processed {len(successful)} files successfully",
|
||||
|
||||
Reference in New Issue
Block a user