leetcrypt
9f73595b20
- Expanded protocol database from 18 → 299 signatures (16.6x increase) - Imported 281 protocols from RTL_433 open-source database (286 total devices) - Created automated import script: scripts/import_rtl433_protocols.py - Generated rtl433_protocols_imported.py with timing/frequency/modulation data - Updated protocol_database.py to include RTL433_PROTOCOLS - All 26 tests passing Breakdown by category: - Weather: 116 protocols - Sensors: 36 protocols - TPMS: 25 protocols - Security: 23 protocols - Home Automation: 18 protocols - Other: 50+ protocols Frequency coverage: - 433.92 MHz: 248 protocols - 315.00 MHz: 32 protocols - 915.00 MHz: 1 protocol This provides comprehensive coverage of Sub-GHz IoT devices for accurate identification from raw RF captures.
371 lines
13 KiB
Python
Executable File
371 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Import Flipper Zero Captures with Random African GPS Coordinates
|
|
|
|
Imports all downloaded .sub files with random GPS coordinates across Africa
|
|
to visualize device distribution on the map.
|
|
|
|
Africa Coverage:
|
|
- Latitude: -35° to 37° (South Africa to Mediterranean)
|
|
- Longitude: -17° to 51° (West coast to East coast)
|
|
|
|
Cities included as hotspots:
|
|
- Cairo, Egypt
|
|
- Lagos, Nigeria
|
|
- Nairobi, Kenya
|
|
- Johannesburg, South Africa
|
|
- Casablanca, Morocco
|
|
- Addis Ababa, Ethiopia
|
|
- Dar es Salaam, Tanzania
|
|
- Khartoum, Sudan
|
|
- Accra, Ghana
|
|
- Kampala, Uganda
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import random
|
|
import hashlib
|
|
from pathlib import Path
|
|
from typing import Tuple, List
|
|
from datetime import datetime, timedelta
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from sqlalchemy.orm import Session
|
|
from loguru import logger
|
|
|
|
from src.database.connection import get_session
|
|
from src.database.models import Capture, Device, CaptureMatch
|
|
from src.parser.sub_parser import parse_sub_file
|
|
from src.matcher.engine import SignatureMatcher
|
|
|
|
# =============================================================================
|
|
# CONFIGURATION
|
|
# =============================================================================
|
|
|
|
DATA_DIR = Path(__file__).parent.parent / "data" / "rf_test_datasets"
|
|
|
|
FLIPPER_DATASETS = [
|
|
DATA_DIR / "FlipperZero-Subghz-DB",
|
|
DATA_DIR / "UberGuidoZ_Flipper",
|
|
DATA_DIR / "Full_Flipper_Database",
|
|
]
|
|
|
|
# Africa bounding box
|
|
AFRICA_LAT_MIN = -35.0 # South Africa
|
|
AFRICA_LAT_MAX = 37.0 # Mediterranean coast
|
|
AFRICA_LON_MIN = -17.0 # West coast (Senegal)
|
|
AFRICA_LON_MAX = 51.0 # East coast (Somalia)
|
|
|
|
# Major African cities (hotspots with higher concentration)
|
|
AFRICAN_CITIES = [
|
|
{"name": "Cairo", "lat": 30.0444, "lon": 31.2357, "weight": 10},
|
|
{"name": "Lagos", "lat": 6.5244, "lon": 3.3792, "weight": 8},
|
|
{"name": "Nairobi", "lat": -1.2921, "lon": 36.8219, "weight": 7},
|
|
{"name": "Johannesburg", "lat": -26.2041, "lon": 28.0473, "weight": 8},
|
|
{"name": "Casablanca", "lat": 33.5731, "lon": -7.5898, "weight": 6},
|
|
{"name": "Addis Ababa", "lat": 9.0320, "lon": 38.7469, "weight": 6},
|
|
{"name": "Dar es Salaam", "lat": -6.7924, "lon": 39.2083, "weight": 5},
|
|
{"name": "Khartoum", "lat": 15.5007, "lon": 32.5599, "weight": 5},
|
|
{"name": "Accra", "lat": 5.6037, "lon": -0.1870, "weight": 5},
|
|
{"name": "Kampala", "lat": 0.3476, "lon": 32.5825, "weight": 5},
|
|
{"name": "Kinshasa", "lat": -4.4419, "lon": 15.2663, "weight": 6},
|
|
{"name": "Luanda", "lat": -8.8368, "lon": 13.2343, "weight": 5},
|
|
{"name": "Dakar", "lat": 14.7167, "lon": -17.4677, "weight": 4},
|
|
{"name": "Cape Town", "lat": -33.9249, "lon": 18.4241, "weight": 6},
|
|
{"name": "Abidjan", "lat": 5.3600, "lon": -4.0083, "weight": 5},
|
|
]
|
|
|
|
|
|
# =============================================================================
|
|
# GPS GENERATION
|
|
# =============================================================================
|
|
|
|
def generate_random_african_gps(use_hotspots: bool = True) -> Tuple[float, float, float]:
|
|
"""
|
|
Generate random GPS coordinates in Africa
|
|
|
|
Returns:
|
|
(latitude, longitude, accuracy_meters)
|
|
"""
|
|
if use_hotspots and random.random() < 0.6: # 60% chance of city hotspot
|
|
# Weight by city population/importance
|
|
weights = [city["weight"] for city in AFRICAN_CITIES]
|
|
city = random.choices(AFRICAN_CITIES, weights=weights, k=1)[0]
|
|
|
|
# Add random offset within ~10km of city center
|
|
lat_offset = random.gauss(0, 0.05) # ~5.5 km std dev
|
|
lon_offset = random.gauss(0, 0.05)
|
|
|
|
latitude = city["lat"] + lat_offset
|
|
longitude = city["lon"] + lon_offset
|
|
accuracy = random.uniform(5.0, 15.0) # Urban GPS accuracy
|
|
|
|
else:
|
|
# Random location anywhere in Africa
|
|
latitude = random.uniform(AFRICA_LAT_MIN, AFRICA_LAT_MAX)
|
|
longitude = random.uniform(AFRICA_LON_MIN, AFRICA_LON_MAX)
|
|
accuracy = random.uniform(10.0, 50.0) # Rural GPS accuracy
|
|
|
|
return latitude, longitude, accuracy
|
|
|
|
|
|
def generate_random_timestamp(days_back: int = 90) -> datetime:
|
|
"""Generate random timestamp within last N days"""
|
|
now = datetime.utcnow()
|
|
delta = timedelta(days=random.randint(0, days_back))
|
|
return now - delta
|
|
|
|
|
|
# =============================================================================
|
|
# IMPORT WITH GPS
|
|
# =============================================================================
|
|
|
|
def import_captures_with_african_gps(
|
|
db: Session,
|
|
limit: int = None,
|
|
skip_existing: bool = True,
|
|
run_matching: bool = True
|
|
) -> dict:
|
|
"""
|
|
Import all .sub files with random African GPS coordinates
|
|
|
|
Returns:
|
|
Statistics dict
|
|
"""
|
|
stats = {
|
|
"files_processed": 0,
|
|
"captures_created": 0,
|
|
"duplicates_skipped": 0,
|
|
"parse_errors": 0,
|
|
"matches_created": 0,
|
|
"cities_covered": set()
|
|
}
|
|
|
|
# Collect all .sub files
|
|
sub_files: List[Path] = []
|
|
for dataset_dir in FLIPPER_DATASETS:
|
|
if not dataset_dir.exists():
|
|
logger.warning(f"Dataset not found: {dataset_dir}")
|
|
continue
|
|
|
|
logger.info(f"Scanning: {dataset_dir}")
|
|
sub_files.extend(dataset_dir.rglob("*.sub"))
|
|
|
|
logger.info(f"Found {len(sub_files)} .sub files total")
|
|
|
|
if limit:
|
|
sub_files = sub_files[:limit]
|
|
logger.info(f"Limited to {limit} files")
|
|
|
|
# Shuffle for geographic randomness
|
|
random.shuffle(sub_files)
|
|
|
|
# Initialize matcher if enabled
|
|
matcher = SignatureMatcher(db) if run_matching else None
|
|
|
|
# Process files
|
|
for idx, sub_file in enumerate(sub_files, 1):
|
|
try:
|
|
if idx % 100 == 0:
|
|
logger.info(f"Progress: {idx}/{len(sub_files)} ({idx/len(sub_files)*100:.1f}%)")
|
|
db.commit() # Commit in batches
|
|
|
|
stats["files_processed"] += 1
|
|
|
|
# Read file
|
|
content = sub_file.read_bytes()
|
|
file_hash = hashlib.sha256(content).hexdigest()
|
|
|
|
# Check for duplicates
|
|
if skip_existing:
|
|
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
|
|
if existing:
|
|
stats["duplicates_skipped"] += 1
|
|
continue
|
|
|
|
# Parse .sub file
|
|
try:
|
|
metadata = parse_sub_file(str(sub_file))
|
|
except Exception as e:
|
|
logger.debug(f"Parse failed: {sub_file.name} - {e}")
|
|
stats["parse_errors"] += 1
|
|
continue
|
|
|
|
# Generate random African GPS
|
|
latitude, longitude, accuracy = generate_random_african_gps()
|
|
|
|
# Track which city region this is near
|
|
for city in AFRICAN_CITIES:
|
|
dist_lat = abs(latitude - city["lat"])
|
|
dist_lon = abs(longitude - city["lon"])
|
|
if dist_lat < 1.0 and dist_lon < 1.0: # Within ~100km
|
|
stats["cities_covered"].add(city["name"])
|
|
break
|
|
|
|
# Generate random timestamp (last 90 days)
|
|
captured_at = generate_random_timestamp(days_back=90)
|
|
|
|
# Store file (simplified - just use relative path)
|
|
storage_path = f"data/uploads/{file_hash[:2]}/{file_hash}.sub"
|
|
|
|
# Ensure directory exists
|
|
os.makedirs(os.path.dirname(storage_path), exist_ok=True)
|
|
|
|
# Write file
|
|
with open(storage_path, 'wb') as f:
|
|
f.write(content)
|
|
|
|
# Create Capture record
|
|
capture = Capture(
|
|
file_hash=file_hash,
|
|
session_id=None,
|
|
user_id=None,
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
altitude=random.uniform(0, 1500), # 0-1500m elevation
|
|
gps_accuracy=accuracy,
|
|
captured_at=captured_at,
|
|
frequency=metadata.frequency,
|
|
modulation=metadata.modulation,
|
|
preset=metadata.preset,
|
|
protocol=metadata.protocol,
|
|
bit_length=metadata.bit_length,
|
|
key_data=metadata.key_data,
|
|
timing_element=metadata.timing_element,
|
|
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
|
|
raw_format=metadata.file_format,
|
|
file_path=storage_path,
|
|
file_size=len(content)
|
|
)
|
|
|
|
db.add(capture)
|
|
db.flush() # Get capture ID
|
|
stats["captures_created"] += 1
|
|
|
|
# Run matching engine
|
|
if matcher and run_matching:
|
|
try:
|
|
match_results = matcher.match(metadata)
|
|
|
|
if match_results:
|
|
# Store top 5 matches
|
|
for match_result in match_results[:5]:
|
|
device = db.query(Device).filter_by(id=match_result.device_id).first()
|
|
if not device:
|
|
continue
|
|
|
|
capture_match = CaptureMatch(
|
|
capture_id=capture.id,
|
|
device_id=device.id,
|
|
confidence=match_result.confidence,
|
|
match_method=match_result.method,
|
|
match_details=match_result.details or {}
|
|
)
|
|
db.add(capture_match)
|
|
stats["matches_created"] += 1
|
|
|
|
# Set best match
|
|
best_match = match_results[0]
|
|
capture.device_id = best_match.device_id
|
|
capture.match_confidence = best_match.confidence
|
|
capture.match_method = best_match.method
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Matching failed for {sub_file.name}: {e}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to process {sub_file}: {e}")
|
|
continue
|
|
|
|
# Final commit
|
|
db.commit()
|
|
|
|
return stats
|
|
|
|
|
|
# =============================================================================
|
|
# MAIN
|
|
# =============================================================================
|
|
|
|
def main():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Import .sub files with random African GPS coordinates"
|
|
)
|
|
parser.add_argument("--limit", type=int, default=None, help="Limit number of files")
|
|
parser.add_argument("--no-matching", action="store_true", help="Skip device matching")
|
|
parser.add_argument("--skip-existing", action="store_true", default=True)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Setup logging
|
|
logger.remove()
|
|
logger.add(sys.stderr, level="INFO")
|
|
logger.add("logs/import_african_gps.log", rotation="10 MB", level="DEBUG")
|
|
|
|
logger.info("=" * 80)
|
|
logger.info("Importing Captures with African GPS Coordinates")
|
|
logger.info("=" * 80)
|
|
|
|
# Setup
|
|
db = get_session()
|
|
|
|
# Set random seed for reproducibility (optional)
|
|
random.seed(42)
|
|
|
|
try:
|
|
stats = import_captures_with_african_gps(
|
|
db=db,
|
|
limit=args.limit,
|
|
skip_existing=args.skip_existing,
|
|
run_matching=not args.no_matching
|
|
)
|
|
|
|
logger.info("\n" + "=" * 80)
|
|
logger.info("Import Complete!")
|
|
logger.info("=" * 80)
|
|
logger.info(f"Files processed: {stats['files_processed']}")
|
|
logger.info(f"Captures created: {stats['captures_created']}")
|
|
logger.info(f"Duplicates skipped: {stats['duplicates_skipped']}")
|
|
logger.info(f"Parse errors: {stats['parse_errors']}")
|
|
logger.info(f"Matches created: {stats['matches_created']}")
|
|
logger.info(f"Cities covered: {len(stats['cities_covered'])}")
|
|
logger.info(f" → {', '.join(sorted(stats['cities_covered']))}")
|
|
|
|
# Database totals
|
|
try:
|
|
total_captures = db.query(Capture).count()
|
|
total_matches = db.query(CaptureMatch).count()
|
|
|
|
logger.info(f"\nDatabase totals:")
|
|
logger.info(f" Total captures: {total_captures}")
|
|
logger.info(f" Total matches: {total_matches}")
|
|
except Exception as e:
|
|
logger.warning(f"Could not query database totals: {e}")
|
|
|
|
# Geographic distribution
|
|
logger.info(f"\nGeographic distribution:")
|
|
result = db.execute("""
|
|
SELECT
|
|
COUNT(*) as count,
|
|
AVG(latitude) as avg_lat,
|
|
AVG(longitude) as avg_lon
|
|
FROM captures
|
|
WHERE latitude BETWEEN -35 AND 37
|
|
AND longitude BETWEEN -17 AND 51
|
|
""").fetchone()
|
|
|
|
if result:
|
|
logger.info(f" Captures in Africa: {result[0]}")
|
|
logger.info(f" Center point: {result[1]:.2f}°, {result[2]:.2f}°")
|
|
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|