Initial commit: Phase 1 & Phase 2 infrastructure complete
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
GigLez - IoT RF Wardriving Platform
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__author__ = "GigLez Contributors"
|
||||
__license__ = "MIT"
|
||||
@@ -0,0 +1,9 @@
|
||||
"""
|
||||
GigLez API Package
|
||||
|
||||
FastAPI application with environment-aware configuration
|
||||
"""
|
||||
|
||||
from .main import app
|
||||
|
||||
__all__ = ['app']
|
||||
@@ -0,0 +1,185 @@
|
||||
"""
|
||||
FastAPI Dependencies
|
||||
|
||||
Shared dependencies for API endpoints
|
||||
"""
|
||||
|
||||
from typing import Generator, Optional
|
||||
from fastapi import Depends, HTTPException, status, Security
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from config.database import get_db_session
|
||||
from config.settings import settings
|
||||
from src.database.models import User
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DATABASE DEPENDENCY
|
||||
# =============================================================================
|
||||
|
||||
def get_db() -> Generator[Session, None, None]:
|
||||
"""
|
||||
Database session dependency
|
||||
|
||||
Yields database session and ensures it's closed after request
|
||||
"""
|
||||
session = get_db_session()
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# AUTHENTICATION DEPENDENCIES
|
||||
# =============================================================================
|
||||
|
||||
# HTTP Bearer token scheme
|
||||
security = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
def get_current_user(
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Security(security),
|
||||
db: Session = Depends(get_db)
|
||||
) -> Optional[User]:
|
||||
"""
|
||||
Get current authenticated user from JWT token
|
||||
|
||||
Args:
|
||||
credentials: Bearer token from Authorization header
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
User object if authenticated, None if auth not required
|
||||
|
||||
Raises:
|
||||
HTTPException: If authentication required but invalid
|
||||
"""
|
||||
|
||||
# If authentication not required (development mode)
|
||||
if not settings.require_auth:
|
||||
# Return anonymous user (ID 0)
|
||||
return db.query(User).filter_by(id=0).first()
|
||||
|
||||
# Authentication required
|
||||
if not credentials:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Authentication required",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
try:
|
||||
# Decode JWT token (implementation in auth.py)
|
||||
from src.api.auth import decode_access_token
|
||||
|
||||
payload = decode_access_token(credentials.credentials)
|
||||
user_id = payload.get("sub")
|
||||
|
||||
if user_id is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid authentication token"
|
||||
)
|
||||
|
||||
# Get user from database
|
||||
user = db.query(User).filter_by(id=int(user_id)).first()
|
||||
|
||||
if user is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="User not found"
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=f"Authentication failed: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def require_auth(user: User = Depends(get_current_user)) -> User:
|
||||
"""
|
||||
Require authentication (always)
|
||||
|
||||
Use this dependency when endpoint must have authentication
|
||||
regardless of global settings
|
||||
|
||||
Args:
|
||||
user: Current user from get_current_user
|
||||
|
||||
Returns:
|
||||
Authenticated user
|
||||
|
||||
Raises:
|
||||
HTTPException: If user not authenticated
|
||||
"""
|
||||
if user is None or user.id == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Authentication required for this endpoint"
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
|
||||
def optional_auth(user: Optional[User] = Depends(get_current_user)) -> Optional[User]:
|
||||
"""
|
||||
Optional authentication
|
||||
|
||||
Returns user if authenticated, None otherwise (no error)
|
||||
|
||||
Args:
|
||||
user: Current user from get_current_user
|
||||
|
||||
Returns:
|
||||
User if authenticated, None otherwise
|
||||
"""
|
||||
return user
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PAGINATION DEPENDENCIES
|
||||
# =============================================================================
|
||||
|
||||
def get_pagination(
|
||||
skip: int = 0,
|
||||
limit: int = 100
|
||||
) -> dict:
|
||||
"""
|
||||
Pagination parameters
|
||||
|
||||
Args:
|
||||
skip: Number of records to skip
|
||||
limit: Maximum number of records to return
|
||||
|
||||
Returns:
|
||||
Dictionary with skip and limit
|
||||
"""
|
||||
# Enforce maximum limit
|
||||
max_limit = 1000
|
||||
if limit > max_limit:
|
||||
limit = max_limit
|
||||
|
||||
return {
|
||||
"skip": skip,
|
||||
"limit": limit
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STORAGE DEPENDENCY
|
||||
# =============================================================================
|
||||
|
||||
def get_storage():
|
||||
"""
|
||||
Get storage backend
|
||||
|
||||
Returns:
|
||||
Storage backend instance (LocalStorage or S3Storage)
|
||||
"""
|
||||
from src.core.storage import get_storage
|
||||
return get_storage()
|
||||
+262
@@ -0,0 +1,262 @@
|
||||
"""
|
||||
GigLez FastAPI Application
|
||||
|
||||
Environment-aware API server supporting both development (Termux) and production (Server) modes
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from contextlib import asynccontextmanager
|
||||
from loguru import logger
|
||||
import time
|
||||
|
||||
from config.settings import settings
|
||||
from config.database import get_db_config
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LIFESPAN MANAGEMENT
|
||||
# =============================================================================
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Application lifespan manager
|
||||
Handles startup and shutdown events
|
||||
"""
|
||||
# Startup
|
||||
logger.info("=" * 80)
|
||||
logger.info("GigLez API Starting")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Display configuration
|
||||
settings.display()
|
||||
|
||||
# Validate configuration
|
||||
errors = settings.validate()
|
||||
if errors:
|
||||
logger.error("Configuration validation failed:")
|
||||
for error in errors:
|
||||
logger.error(f" ❌ {error}")
|
||||
raise RuntimeError("Invalid configuration")
|
||||
|
||||
# Test database connection
|
||||
db_config = get_db_config()
|
||||
if not db_config.test_connection():
|
||||
raise RuntimeError("Database connection failed")
|
||||
|
||||
if not db_config.test_postgis():
|
||||
raise RuntimeError("PostGIS extension not available")
|
||||
|
||||
logger.success("✅ Database connection established")
|
||||
logger.success("✅ PostGIS extension available")
|
||||
|
||||
# Initialize storage backend
|
||||
from src.core.storage import get_storage_backend
|
||||
storage = get_storage_backend()
|
||||
logger.success(f"✅ Storage backend initialized: {type(storage).__name__}")
|
||||
|
||||
# Initialize background task system
|
||||
if settings.use_celery:
|
||||
logger.info("Using Celery for background tasks")
|
||||
# Celery app will be imported when needed
|
||||
else:
|
||||
logger.info("Using synchronous background tasks")
|
||||
|
||||
logger.success("=" * 80)
|
||||
logger.success(f"🚀 GigLez API ready on {settings.api_host}:{settings.api_port}")
|
||||
logger.success(f"📡 Mode: {settings.mode.value}")
|
||||
logger.success("=" * 80)
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
logger.info("=" * 80)
|
||||
logger.info("GigLez API Shutting Down")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Close database connections
|
||||
db_config.close()
|
||||
logger.success("✅ Database connections closed")
|
||||
|
||||
logger.info("Shutdown complete")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# APPLICATION INSTANCE
|
||||
# =============================================================================
|
||||
|
||||
app = FastAPI(
|
||||
title="GigLez API",
|
||||
description="IoT RF Device Mapping Platform - Wigle for Sub-GHz Signals",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
docs_url="/docs" if settings.debug_endpoints else None,
|
||||
redoc_url="/redoc" if settings.debug_endpoints else None,
|
||||
openapi_url="/openapi.json" if settings.debug_endpoints else None,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MIDDLEWARE
|
||||
# =============================================================================
|
||||
|
||||
# CORS Middleware
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
expose_headers=["X-Total-Count", "X-Page-Size"],
|
||||
)
|
||||
|
||||
# GZip compression
|
||||
app.add_middleware(GZipMiddleware, minimum_size=1000)
|
||||
|
||||
|
||||
# Request timing middleware
|
||||
@app.middleware("http")
|
||||
async def add_process_time_header(request: Request, call_next):
|
||||
"""Add request processing time header"""
|
||||
start_time = time.time()
|
||||
response = await call_next(request)
|
||||
process_time = time.time() - start_time
|
||||
response.headers["X-Process-Time"] = f"{process_time:.4f}"
|
||||
return response
|
||||
|
||||
|
||||
# Request logging middleware
|
||||
@app.middleware("http")
|
||||
async def log_requests(request: Request, call_next):
|
||||
"""Log all requests"""
|
||||
logger.info(f"{request.method} {request.url.path}")
|
||||
response = await call_next(request)
|
||||
logger.info(f"{request.method} {request.url.path} - {response.status_code}")
|
||||
return response
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# EXCEPTION HANDLERS
|
||||
# =============================================================================
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def global_exception_handler(request: Request, exc: Exception):
|
||||
"""Global exception handler"""
|
||||
logger.error(f"Unhandled exception: {exc}", exc_info=True)
|
||||
|
||||
if settings.debug_errors:
|
||||
# Detailed error in development
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"error": "Internal Server Error",
|
||||
"detail": str(exc),
|
||||
"type": type(exc).__name__
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Generic error in production
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": "Internal Server Error"}
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ROOT ENDPOINTS
|
||||
# =============================================================================
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""Root endpoint - API information"""
|
||||
return {
|
||||
"name": "GigLez API",
|
||||
"version": "1.0.0",
|
||||
"description": "IoT RF Device Mapping Platform",
|
||||
"mode": settings.mode.value,
|
||||
"docs": "/docs" if settings.debug_endpoints else "disabled",
|
||||
"status": "operational"
|
||||
}
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
"""Health check endpoint"""
|
||||
# Test database connection
|
||||
db_config = get_db_config()
|
||||
db_healthy = db_config.test_connection()
|
||||
|
||||
return {
|
||||
"status": "healthy" if db_healthy else "unhealthy",
|
||||
"database": "connected" if db_healthy else "disconnected",
|
||||
"mode": settings.mode.value,
|
||||
"hardware_enabled": settings.enable_hardware
|
||||
}
|
||||
|
||||
|
||||
@app.get("/version")
|
||||
async def version():
|
||||
"""Version information"""
|
||||
return {
|
||||
"version": "1.0.0",
|
||||
"mode": settings.mode.value,
|
||||
"api": "v1"
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# IMPORT ROUTERS
|
||||
# =============================================================================
|
||||
|
||||
# Import and register route modules
|
||||
from src.api.routes import captures, devices, query, stats
|
||||
|
||||
# Register routers
|
||||
app.include_router(captures.router, prefix="/api/v1", tags=["captures"])
|
||||
app.include_router(devices.router, prefix="/api/v1", tags=["devices"])
|
||||
app.include_router(query.router, prefix="/api/v1", tags=["query"])
|
||||
app.include_router(stats.router, prefix="/api/v1", tags=["statistics"])
|
||||
|
||||
# Hardware endpoints (development mode only)
|
||||
if settings.enable_hardware:
|
||||
from src.api.routes import hardware
|
||||
app.include_router(hardware.router, prefix="/api/v1", tags=["hardware"])
|
||||
logger.info("✅ Hardware endpoints enabled")
|
||||
|
||||
|
||||
# Debug endpoints (development mode only)
|
||||
if settings.debug_endpoints:
|
||||
from src.api.routes import debug
|
||||
app.include_router(debug.router, prefix="/api/debug", tags=["debug"])
|
||||
logger.info("✅ Debug endpoints enabled")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STARTUP MESSAGE
|
||||
# =============================================================================
|
||||
|
||||
logger.info("FastAPI application initialized")
|
||||
logger.info(f"Mode: {settings.mode.value}")
|
||||
logger.info(f"Authentication: {'Required' if settings.require_auth else 'Optional'}")
|
||||
logger.info(f"CORS Origins: {settings.cors_origins}")
|
||||
logger.info(f"Storage: {settings.storage_type.value}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# RUN WITH UVICORN (for development)
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"src.api.main:app",
|
||||
host=settings.api_host,
|
||||
port=settings.api_port,
|
||||
reload=settings.auto_reload,
|
||||
log_level=settings.log_level.lower(),
|
||||
access_log=True,
|
||||
)
|
||||
@@ -0,0 +1,5 @@
|
||||
"""
|
||||
GigLez API Routes
|
||||
|
||||
Route modules for different API endpoints
|
||||
"""
|
||||
@@ -0,0 +1,267 @@
|
||||
"""
|
||||
Capture Upload Endpoints
|
||||
|
||||
Handles .sub file uploads with GPS manifests
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
from loguru import logger
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from src.api.dependencies import get_db, optional_auth, get_storage
|
||||
from src.database.models import User, Capture, Session as CaptureSession
|
||||
from src.parser.sub_parser import parse_sub_file
|
||||
from src.gps.validator import validate_gps_coordinates
|
||||
from config.settings import settings
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# UPLOAD ENDPOINT
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/captures/upload")
|
||||
async def upload_captures(
|
||||
manifest: str = Form(..., description="JSON manifest with GPS data"),
|
||||
files: List[UploadFile] = File(..., description=".sub files to upload"),
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(optional_auth),
|
||||
storage = Depends(get_storage)
|
||||
):
|
||||
"""
|
||||
Upload RF captures with GPS coordinates
|
||||
|
||||
**Manifest Format** (JSON):
|
||||
```json
|
||||
{
|
||||
"session_uuid": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"captures": [
|
||||
{
|
||||
"filename": "capture_001.sub",
|
||||
"latitude": 40.7128,
|
||||
"longitude": -74.0060,
|
||||
"accuracy": 5.0,
|
||||
"altitude": 10.5,
|
||||
"timestamp": "2026-01-12T10:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Parse and validate manifest
|
||||
2. Compute SHA256 hash for each file (deduplication)
|
||||
3. Validate GPS coordinates
|
||||
4. Parse .sub file metadata
|
||||
5. Store file in storage backend
|
||||
6. Save capture to database
|
||||
7. Return upload summary
|
||||
|
||||
Returns:
|
||||
Upload summary with processed captures
|
||||
"""
|
||||
|
||||
try:
|
||||
# Parse manifest
|
||||
manifest_data = json.loads(manifest)
|
||||
|
||||
# Extract session UUID
|
||||
session_uuid = manifest_data.get("session_uuid")
|
||||
captures_manifest = manifest_data.get("captures", [])
|
||||
|
||||
logger.info(f"Upload request: {len(files)} files, session={session_uuid}")
|
||||
|
||||
# Get or create session
|
||||
capture_session = None
|
||||
if session_uuid:
|
||||
capture_session = db.query(CaptureSession).filter_by(
|
||||
session_uuid=session_uuid
|
||||
).first()
|
||||
|
||||
if not capture_session:
|
||||
# Create new session
|
||||
capture_session = CaptureSession(
|
||||
session_uuid=session_uuid,
|
||||
user_id=user.id if user else None,
|
||||
started_at=datetime.utcnow()
|
||||
)
|
||||
db.add(capture_session)
|
||||
db.flush() # Get session ID
|
||||
|
||||
# Process files
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
# Create filename to manifest mapping
|
||||
manifest_map = {c['filename']: c for c in captures_manifest}
|
||||
|
||||
for file in files:
|
||||
try:
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
|
||||
# Compute SHA256 hash
|
||||
file_hash = hashlib.sha256(content).hexdigest()
|
||||
|
||||
# Check if already exists (deduplication)
|
||||
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
|
||||
if existing:
|
||||
results.append({
|
||||
"filename": file.filename,
|
||||
"status": "duplicate",
|
||||
"file_hash": file_hash
|
||||
})
|
||||
continue
|
||||
|
||||
# Get manifest entry for this file
|
||||
manifest_entry = manifest_map.get(file.filename)
|
||||
if not manifest_entry:
|
||||
errors.append({
|
||||
"filename": file.filename,
|
||||
"error": "Missing manifest entry"
|
||||
})
|
||||
continue
|
||||
|
||||
# Validate GPS coordinates
|
||||
latitude = float(manifest_entry['latitude'])
|
||||
longitude = float(manifest_entry['longitude'])
|
||||
accuracy = manifest_entry.get('accuracy')
|
||||
|
||||
if not validate_gps_coordinates(latitude, longitude, accuracy):
|
||||
errors.append({
|
||||
"filename": file.filename,
|
||||
"error": f"Invalid GPS coordinates: ({latitude}, {longitude})"
|
||||
})
|
||||
continue
|
||||
|
||||
# Parse .sub file
|
||||
# Save content temporarily to parse
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix='.sub') as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
metadata = parse_sub_file(tmp_path)
|
||||
except Exception as e:
|
||||
errors.append({
|
||||
"filename": file.filename,
|
||||
"error": f"Failed to parse .sub file: {e}"
|
||||
})
|
||||
continue
|
||||
finally:
|
||||
import os
|
||||
os.unlink(tmp_path)
|
||||
|
||||
# Store file in storage backend
|
||||
storage_path = storage.save_file(
|
||||
file_hash=file_hash,
|
||||
content=content,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
# Create capture record
|
||||
capture = Capture(
|
||||
file_hash=file_hash,
|
||||
session_id=capture_session.id if capture_session else None,
|
||||
user_id=user.id if user else None,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
altitude=manifest_entry.get('altitude'),
|
||||
gps_accuracy=accuracy,
|
||||
captured_at=datetime.fromisoformat(
|
||||
manifest_entry['timestamp'].replace('Z', '+00:00')
|
||||
),
|
||||
frequency=metadata.frequency,
|
||||
rssi=None, # Not in .sub file typically
|
||||
modulation=metadata.modulation,
|
||||
preset=metadata.preset,
|
||||
protocol=metadata.protocol,
|
||||
bit_length=metadata.bit_length,
|
||||
key_data=metadata.key_data,
|
||||
timing_element=metadata.timing_element,
|
||||
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
|
||||
raw_format=metadata.file_format,
|
||||
file_path=storage_path,
|
||||
file_size=len(content)
|
||||
)
|
||||
|
||||
db.add(capture)
|
||||
|
||||
results.append({
|
||||
"filename": file.filename,
|
||||
"status": "uploaded",
|
||||
"file_hash": file_hash,
|
||||
"frequency": metadata.frequency,
|
||||
"protocol": metadata.protocol
|
||||
})
|
||||
|
||||
logger.debug(f"Capture uploaded: {file.filename} -> {file_hash}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process file {file.filename}: {e}")
|
||||
errors.append({
|
||||
"filename": file.filename,
|
||||
"error": str(e)
|
||||
})
|
||||
|
||||
# Commit all captures
|
||||
db.commit()
|
||||
|
||||
logger.success(f"Upload complete: {len(results)} processed, {len(errors)} errors")
|
||||
|
||||
return {
|
||||
"session_uuid": session_uuid,
|
||||
"uploaded": len([r for r in results if r['status'] == 'uploaded']),
|
||||
"duplicates": len([r for r in results if r['status'] == 'duplicate']),
|
||||
"errors": len(errors),
|
||||
"results": results,
|
||||
"error_details": errors if errors else None
|
||||
}
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON manifest: {e}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Upload failed: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Upload failed: {e}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CAPTURE QUERY ENDPOINT
|
||||
# =============================================================================
|
||||
|
||||
@router.get("/captures/{file_hash}")
|
||||
async def get_capture(
|
||||
file_hash: str,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get capture details by file hash
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of capture file
|
||||
|
||||
Returns:
|
||||
Capture details including metadata and device match
|
||||
"""
|
||||
capture = db.query(Capture).filter_by(file_hash=file_hash).first()
|
||||
|
||||
if not capture:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Capture not found: {file_hash}"
|
||||
)
|
||||
|
||||
return capture.to_dict()
|
||||
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
Debug Endpoints (Development Mode Only)
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from config.settings import settings
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config():
|
||||
"""Get current configuration"""
|
||||
return {
|
||||
"mode": settings.mode.value,
|
||||
"storage": settings.storage_type.value,
|
||||
"hardware_enabled": settings.enable_hardware
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
Device Catalog Endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from src.api.dependencies import get_db
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/devices")
|
||||
async def list_devices(db: Session = Depends(get_db)):
|
||||
"""List all known devices"""
|
||||
# TODO: Implement device listing
|
||||
return {"devices": []}
|
||||
|
||||
@router.get("/devices/{device_id}")
|
||||
async def get_device(device_id: int, db: Session = Depends(get_db)):
|
||||
"""Get device details"""
|
||||
# TODO: Implement device details
|
||||
return {"device_id": device_id}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Hardware Control Endpoints (Development Mode Only)
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.post("/hardware/scan")
|
||||
async def start_scan():
|
||||
"""Start RF scanning (Termux mode only)"""
|
||||
# TODO: Implement T-Embed scanning
|
||||
return {"status": "scanning"}
|
||||
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
Query Endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from src.api.dependencies import get_db
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/query")
|
||||
async def query_captures(db: Session = Depends(get_db)):
|
||||
"""Query captures by location/time/protocol"""
|
||||
# TODO: Implement geospatial queries
|
||||
return {"captures": []}
|
||||
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
Statistics Endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from src.api.dependencies import get_db
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/stats")
|
||||
async def get_stats(db: Session = Depends(get_db)):
|
||||
"""Get platform statistics"""
|
||||
# TODO: Implement statistics
|
||||
return {"total_captures": 0}
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
RF signal capture module for T-Embed communication
|
||||
"""
|
||||
|
||||
from .tembed import TEmbedController, AsyncTEmbedController
|
||||
from .scanner import SignalScanner
|
||||
|
||||
__all__ = ['TEmbedController', 'AsyncTEmbedController', 'SignalScanner']
|
||||
@@ -0,0 +1,317 @@
|
||||
"""
|
||||
LilyGo T-Embed communication controller
|
||||
"""
|
||||
|
||||
import serial
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
from typing import Dict, Any, Optional, Callable, List
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class TEmbedController:
|
||||
"""Synchronous T-Embed controller"""
|
||||
|
||||
def __init__(self, port: str = '/dev/ttyUSB0', baudrate: int = 115200):
|
||||
"""
|
||||
Initialize T-Embed controller
|
||||
|
||||
Args:
|
||||
port: Serial port path
|
||||
baudrate: Serial baudrate (default 115200)
|
||||
"""
|
||||
self.port = port
|
||||
self.baudrate = baudrate
|
||||
self.serial = None
|
||||
self.request_id = 0
|
||||
self.connect()
|
||||
|
||||
def connect(self):
|
||||
"""Establish serial connection"""
|
||||
try:
|
||||
self.serial = serial.Serial(self.port, self.baudrate, timeout=1)
|
||||
time.sleep(2) # Wait for device reset
|
||||
logger.info(f"Connected to T-Embed on {self.port}")
|
||||
except serial.SerialException as e:
|
||||
logger.error(f"Failed to connect to T-Embed: {e}")
|
||||
raise
|
||||
|
||||
def disconnect(self):
|
||||
"""Close serial connection"""
|
||||
if self.serial and self.serial.is_open:
|
||||
self.serial.close()
|
||||
logger.info("Disconnected from T-Embed")
|
||||
|
||||
def send_command(self, cmd: str, params: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Send command to T-Embed and wait for response
|
||||
|
||||
Args:
|
||||
cmd: Command name
|
||||
params: Command parameters
|
||||
|
||||
Returns:
|
||||
Response dictionary or None
|
||||
"""
|
||||
self.request_id += 1
|
||||
message = {
|
||||
'cmd': cmd,
|
||||
'params': params or {},
|
||||
'id': self.request_id
|
||||
}
|
||||
|
||||
try:
|
||||
# Send command
|
||||
self.serial.write(json.dumps(message).encode() + b'\n')
|
||||
logger.debug(f"Sent command: {cmd} (id={self.request_id})")
|
||||
|
||||
# Wait for response
|
||||
line = self.serial.readline().decode().strip()
|
||||
if line:
|
||||
response = json.loads(line)
|
||||
logger.debug(f"Received response: {response.get('status')}")
|
||||
return response
|
||||
|
||||
except (serial.SerialException, json.JSONDecodeError) as e:
|
||||
logger.error(f"Command failed: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def start_scan(self, frequencies: List[int], duration: int = 60,
|
||||
modulation: str = 'AUTO', rssi_threshold: int = -90) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Start scanning for signals
|
||||
|
||||
Args:
|
||||
frequencies: List of frequencies in Hz
|
||||
duration: Scan duration in seconds (0 = continuous)
|
||||
modulation: Modulation type (OOK, FSK, AUTO)
|
||||
rssi_threshold: Minimum signal strength in dBm
|
||||
|
||||
Returns:
|
||||
Response with scan_id
|
||||
"""
|
||||
return self.send_command('SCAN', {
|
||||
'frequencies': frequencies,
|
||||
'duration': duration,
|
||||
'modulation': modulation,
|
||||
'rssi_threshold': rssi_threshold
|
||||
})
|
||||
|
||||
def stop_scan(self, scan_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Stop active scan"""
|
||||
return self.send_command('STOP_SCAN', {'scan_id': scan_id})
|
||||
|
||||
def capture_signal(self, frequency: int, duration: int = 5,
|
||||
format: str = 'BOTH') -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Capture signal at specific frequency
|
||||
|
||||
Args:
|
||||
frequency: Frequency in Hz
|
||||
duration: Capture duration in seconds
|
||||
format: Output format (RAW, DECODED, BOTH)
|
||||
|
||||
Returns:
|
||||
Capture data with protocol and raw signal
|
||||
"""
|
||||
return self.send_command('CAPTURE', {
|
||||
'frequency': frequency,
|
||||
'duration': duration,
|
||||
'format': format
|
||||
})
|
||||
|
||||
def get_status(self) -> Optional[Dict[str, Any]]:
|
||||
"""Get device status"""
|
||||
return self.send_command('STATUS')
|
||||
|
||||
def configure(self, **settings) -> Optional[Dict[str, Any]]:
|
||||
"""Configure device settings"""
|
||||
return self.send_command('CONFIG', settings)
|
||||
|
||||
def list_files(self, path: str = '/sd/giglez/', pattern: str = '*.sub') -> Optional[Dict[str, Any]]:
|
||||
"""List files on SD card"""
|
||||
return self.send_command('LIST_FILES', {
|
||||
'path': path,
|
||||
'pattern': pattern
|
||||
})
|
||||
|
||||
def get_file(self, path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Retrieve file from SD card"""
|
||||
return self.send_command('GET_FILE', {'path': path})
|
||||
|
||||
def replay(self, file_path: str, repeat: int = 1, delay_ms: int = 100) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Replay a .sub file
|
||||
|
||||
Args:
|
||||
file_path: Path to .sub file on SD card
|
||||
repeat: Number of times to repeat
|
||||
delay_ms: Delay between repeats in milliseconds
|
||||
|
||||
Returns:
|
||||
Transmission result
|
||||
"""
|
||||
return self.send_command('REPLAY', {
|
||||
'file': file_path,
|
||||
'repeat': repeat,
|
||||
'delay_ms': delay_ms
|
||||
})
|
||||
|
||||
def listen_events(self, callback: Callable[[Dict[str, Any]], None], timeout: Optional[int] = None):
|
||||
"""
|
||||
Listen for event notifications
|
||||
|
||||
Args:
|
||||
callback: Function to call for each event
|
||||
timeout: Optional timeout in seconds
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
if timeout and (time.time() - start_time) > timeout:
|
||||
break
|
||||
|
||||
try:
|
||||
line = self.serial.readline().decode().strip()
|
||||
if line:
|
||||
data = json.loads(line)
|
||||
if 'event' in data:
|
||||
callback(data)
|
||||
except (json.JSONDecodeError, KeyboardInterrupt):
|
||||
break
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.disconnect()
|
||||
|
||||
|
||||
class AsyncTEmbedController:
|
||||
"""Asynchronous T-Embed controller for better performance"""
|
||||
|
||||
def __init__(self, port: str = '/dev/ttyUSB0', baudrate: int = 115200):
|
||||
self.port = port
|
||||
self.baudrate = baudrate
|
||||
self.serial = None
|
||||
self.request_id = 0
|
||||
self.pending_requests: Dict[int, asyncio.Future] = {}
|
||||
self.event_callbacks: List[Callable] = []
|
||||
|
||||
async def connect(self):
|
||||
"""Establish async serial connection"""
|
||||
try:
|
||||
import aioserial
|
||||
self.serial = aioserial.AioSerial(port=self.port, baudrate=self.baudrate)
|
||||
await asyncio.sleep(2)
|
||||
logger.info(f"Connected to T-Embed on {self.port}")
|
||||
|
||||
# Start background listener
|
||||
asyncio.create_task(self._listen())
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to T-Embed: {e}")
|
||||
raise
|
||||
|
||||
async def disconnect(self):
|
||||
"""Close async serial connection"""
|
||||
if self.serial:
|
||||
self.serial.close()
|
||||
logger.info("Disconnected from T-Embed")
|
||||
|
||||
async def send_command(self, cmd: str, params: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = 10) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Send command and wait for response asynchronously
|
||||
|
||||
Args:
|
||||
cmd: Command name
|
||||
params: Command parameters
|
||||
timeout: Response timeout in seconds
|
||||
|
||||
Returns:
|
||||
Response dictionary
|
||||
"""
|
||||
self.request_id += 1
|
||||
message = {
|
||||
'cmd': cmd,
|
||||
'params': params or {},
|
||||
'id': self.request_id
|
||||
}
|
||||
|
||||
try:
|
||||
# Send command
|
||||
await self.serial.write_async(json.dumps(message).encode() + b'\n')
|
||||
logger.debug(f"Sent command: {cmd} (id={self.request_id})")
|
||||
|
||||
# Create future for response
|
||||
future = asyncio.Future()
|
||||
self.pending_requests[self.request_id] = future
|
||||
|
||||
# Wait for response with timeout
|
||||
response = await asyncio.wait_for(future, timeout=timeout)
|
||||
return response
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"Command timeout: {cmd}")
|
||||
del self.pending_requests[self.request_id]
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Command failed: {e}")
|
||||
return None
|
||||
|
||||
async def _listen(self):
|
||||
"""Background task to listen for responses and events"""
|
||||
while self.serial and not self.serial.closed:
|
||||
try:
|
||||
line = await self.serial.readline_async()
|
||||
if line:
|
||||
data = json.loads(line.decode().strip())
|
||||
|
||||
# Handle response
|
||||
if 'id' in data and data['id'] in self.pending_requests:
|
||||
self.pending_requests[data['id']].set_result(data)
|
||||
del self.pending_requests[data['id']]
|
||||
|
||||
# Handle event
|
||||
elif 'event' in data:
|
||||
for callback in self.event_callbacks:
|
||||
await callback(data)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"Listen error: {e}")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
def on_event(self, callback: Callable[[Dict[str, Any]], None]):
|
||||
"""Register event callback"""
|
||||
self.event_callbacks.append(callback)
|
||||
|
||||
async def start_scan(self, frequencies: List[int], **kwargs) -> Optional[Dict[str, Any]]:
|
||||
"""Start scanning (async)"""
|
||||
params = {'frequencies': frequencies, **kwargs}
|
||||
return await self.send_command('SCAN', params)
|
||||
|
||||
async def stop_scan(self, scan_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Stop scan (async)"""
|
||||
return await self.send_command('STOP_SCAN', {'scan_id': scan_id})
|
||||
|
||||
async def capture_signal(self, frequency: int, **kwargs) -> Optional[Dict[str, Any]]:
|
||||
"""Capture signal (async)"""
|
||||
params = {'frequency': frequency, **kwargs}
|
||||
return await self.send_command('CAPTURE', params)
|
||||
|
||||
async def get_status(self) -> Optional[Dict[str, Any]]:
|
||||
"""Get device status (async)"""
|
||||
return await self.send_command('STATUS')
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.connect()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
await self.disconnect()
|
||||
@@ -0,0 +1,5 @@
|
||||
"""
|
||||
GigLez Core Package
|
||||
|
||||
Shared core functionality for both deployment modes
|
||||
"""
|
||||
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
GigLez Storage Package
|
||||
|
||||
Storage abstraction supporting filesystem and S3-compatible object storage
|
||||
"""
|
||||
|
||||
from .base import StorageBackend
|
||||
from .filesystem import LocalStorage
|
||||
from .s3 import S3Storage
|
||||
from .factory import get_storage_backend
|
||||
|
||||
__all__ = [
|
||||
'StorageBackend',
|
||||
'LocalStorage',
|
||||
'S3Storage',
|
||||
'get_storage_backend'
|
||||
]
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
Storage Backend Base Class
|
||||
|
||||
Abstract interface for storage implementations
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, BinaryIO
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class StorageBackend(ABC):
|
||||
"""
|
||||
Abstract base class for storage backends
|
||||
|
||||
Implementations:
|
||||
- LocalStorage: Filesystem storage (development)
|
||||
- S3Storage: S3-compatible object storage (production)
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def save_file(self, file_hash: str, content: bytes, filename: Optional[str] = None) -> str:
|
||||
"""
|
||||
Save file to storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file (used as key)
|
||||
content: File content as bytes
|
||||
filename: Original filename (optional, for metadata)
|
||||
|
||||
Returns:
|
||||
Storage path/URL where file was saved
|
||||
|
||||
Raises:
|
||||
StorageError: If save fails
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_file(self, file_hash: str) -> bytes:
|
||||
"""
|
||||
Retrieve file from storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
File content as bytes
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
StorageError: If retrieval fails
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def file_exists(self, file_hash: str) -> bool:
|
||||
"""
|
||||
Check if file exists in storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
True if file exists, False otherwise
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def delete_file(self, file_hash: str) -> bool:
|
||||
"""
|
||||
Delete file from storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
True if deleted, False if file didn't exist
|
||||
|
||||
Raises:
|
||||
StorageError: If deletion fails
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_file_url(self, file_hash: str) -> str:
|
||||
"""
|
||||
Get URL for accessing file
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
URL string (file:// for local, https:// for S3)
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_file_path(self, file_hash: str) -> str:
|
||||
"""
|
||||
Get storage path for file hash
|
||||
|
||||
Uses content-addressed storage pattern: first 2+2 chars for sharding
|
||||
Example: abc123...def -> ab/c1/abc123...def.sub
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash (64 hex characters)
|
||||
|
||||
Returns:
|
||||
Relative storage path
|
||||
"""
|
||||
if len(file_hash) < 4:
|
||||
raise ValueError(f"Invalid file hash: {file_hash}")
|
||||
|
||||
# Shard by first 2+2 characters (prevents too many files in one directory)
|
||||
shard1 = file_hash[:2]
|
||||
shard2 = file_hash[2:4]
|
||||
filename = f"{file_hash}.sub"
|
||||
|
||||
return f"{shard1}/{shard2}/{filename}"
|
||||
|
||||
|
||||
class StorageError(Exception):
|
||||
"""Base exception for storage errors"""
|
||||
pass
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
Storage Backend Factory
|
||||
|
||||
Creates appropriate storage backend based on configuration
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from config.settings import settings
|
||||
from .base import StorageBackend
|
||||
from .filesystem import LocalStorage
|
||||
from .s3 import S3Storage
|
||||
|
||||
|
||||
def get_storage_backend() -> StorageBackend:
|
||||
"""
|
||||
Get storage backend based on configuration
|
||||
|
||||
Returns:
|
||||
StorageBackend instance (LocalStorage or S3Storage)
|
||||
|
||||
Raises:
|
||||
ValueError: If storage type is invalid
|
||||
"""
|
||||
|
||||
if settings.use_local_storage:
|
||||
logger.info(f"Using LocalStorage: {settings.storage_path}")
|
||||
return LocalStorage(base_path=settings.storage_path)
|
||||
|
||||
elif settings.use_s3_storage:
|
||||
logger.info(f"Using S3Storage: bucket={settings.storage_bucket}")
|
||||
|
||||
if not settings.aws_access_key_id or not settings.aws_secret_access_key:
|
||||
raise ValueError("AWS credentials not configured for S3 storage")
|
||||
|
||||
return S3Storage(
|
||||
bucket=settings.storage_bucket,
|
||||
access_key=settings.aws_access_key_id,
|
||||
secret_key=settings.aws_secret_access_key,
|
||||
endpoint=settings.s3_endpoint,
|
||||
region=settings.aws_region
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown storage type: {settings.storage_type}")
|
||||
|
||||
|
||||
# Create a singleton instance for convenience
|
||||
_storage_instance = None
|
||||
|
||||
|
||||
def get_storage() -> StorageBackend:
|
||||
"""
|
||||
Get singleton storage instance
|
||||
|
||||
Returns:
|
||||
Global storage backend instance
|
||||
"""
|
||||
global _storage_instance
|
||||
|
||||
if _storage_instance is None:
|
||||
_storage_instance = get_storage_backend()
|
||||
|
||||
return _storage_instance
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Local Filesystem Storage Backend
|
||||
|
||||
For development mode (Termux) - stores files in local directory
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
from .base import StorageBackend, StorageError
|
||||
|
||||
|
||||
class LocalStorage(StorageBackend):
|
||||
"""
|
||||
Local filesystem storage implementation
|
||||
|
||||
Stores files in organized directory structure:
|
||||
storage/
|
||||
├── ab/
|
||||
│ ├── c1/
|
||||
│ │ └── abc123...def.sub
|
||||
"""
|
||||
|
||||
def __init__(self, base_path: str = "./storage"):
|
||||
"""
|
||||
Initialize local storage
|
||||
|
||||
Args:
|
||||
base_path: Base directory for file storage
|
||||
"""
|
||||
self.base_path = Path(base_path)
|
||||
|
||||
# Create base directory if it doesn't exist
|
||||
self.base_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"LocalStorage initialized at: {self.base_path.absolute()}")
|
||||
|
||||
def save_file(self, file_hash: str, content: bytes, filename: Optional[str] = None) -> str:
|
||||
"""Save file to local filesystem"""
|
||||
try:
|
||||
# Get storage path
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
# Create parent directories
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write file
|
||||
with open(full_path, 'wb') as f:
|
||||
f.write(content)
|
||||
|
||||
logger.debug(f"File saved: {relative_path} ({len(content)} bytes)")
|
||||
|
||||
return str(relative_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save file {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to save file: {e}")
|
||||
|
||||
def get_file(self, file_hash: str) -> bytes:
|
||||
"""Retrieve file from local filesystem"""
|
||||
try:
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
if not full_path.exists():
|
||||
raise FileNotFoundError(f"File not found: {file_hash}")
|
||||
|
||||
with open(full_path, 'rb') as f:
|
||||
content = f.read()
|
||||
|
||||
logger.debug(f"File retrieved: {relative_path} ({len(content)} bytes)")
|
||||
|
||||
return content
|
||||
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to retrieve file {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to retrieve file: {e}")
|
||||
|
||||
def file_exists(self, file_hash: str) -> bool:
|
||||
"""Check if file exists"""
|
||||
try:
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
return full_path.exists()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check file existence {file_hash}: {e}")
|
||||
return False
|
||||
|
||||
def delete_file(self, file_hash: str) -> bool:
|
||||
"""Delete file from filesystem"""
|
||||
try:
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
if not full_path.exists():
|
||||
return False
|
||||
|
||||
full_path.unlink()
|
||||
logger.debug(f"File deleted: {relative_path}")
|
||||
|
||||
# Clean up empty parent directories
|
||||
try:
|
||||
full_path.parent.rmdir() # Only works if empty
|
||||
full_path.parent.parent.rmdir() # Only works if empty
|
||||
except OSError:
|
||||
pass # Directory not empty, that's fine
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to delete file: {e}")
|
||||
|
||||
def get_file_url(self, file_hash: str) -> str:
|
||||
"""Get file:// URL for local file"""
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
return f"file://{full_path.absolute()}"
|
||||
|
||||
def get_storage_stats(self) -> dict:
|
||||
"""
|
||||
Get storage statistics
|
||||
|
||||
Returns:
|
||||
Dictionary with storage stats
|
||||
"""
|
||||
total_files = 0
|
||||
total_size = 0
|
||||
|
||||
try:
|
||||
for file_path in self.base_path.rglob("*.sub"):
|
||||
total_files += 1
|
||||
total_size += file_path.stat().st_size
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"base_path": str(self.base_path.absolute())
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get storage stats: {e}")
|
||||
return {
|
||||
"error": str(e)
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
"""
|
||||
S3-Compatible Object Storage Backend
|
||||
|
||||
For production mode - stores files in S3 or MinIO
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
from .base import StorageBackend, StorageError
|
||||
|
||||
|
||||
class S3Storage(StorageBackend):
|
||||
"""
|
||||
S3-compatible object storage implementation
|
||||
|
||||
Supports:
|
||||
- AWS S3
|
||||
- MinIO
|
||||
- Any S3-compatible storage
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bucket: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
endpoint: Optional[str] = None,
|
||||
region: str = "us-east-1"
|
||||
):
|
||||
"""
|
||||
Initialize S3 storage
|
||||
|
||||
Args:
|
||||
bucket: S3 bucket name
|
||||
access_key: AWS access key ID
|
||||
secret_key: AWS secret access key
|
||||
endpoint: Custom endpoint (for MinIO, etc.)
|
||||
region: AWS region
|
||||
"""
|
||||
self.bucket = bucket
|
||||
self.access_key = access_key
|
||||
self.secret_key = secret_key
|
||||
self.endpoint = endpoint
|
||||
self.region = region
|
||||
|
||||
# Import boto3 lazily (only in production)
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
self.boto3 = boto3
|
||||
self.ClientError = ClientError
|
||||
|
||||
# Create S3 client
|
||||
self.s3_client = boto3.client(
|
||||
's3',
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
endpoint_url=endpoint,
|
||||
region_name=region
|
||||
)
|
||||
|
||||
# Verify bucket exists
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=bucket)
|
||||
logger.info(f"S3Storage initialized: bucket={bucket}, region={region}")
|
||||
except self.ClientError:
|
||||
logger.error(f"S3 bucket not accessible: {bucket}")
|
||||
raise StorageError(f"S3 bucket not accessible: {bucket}")
|
||||
|
||||
except ImportError:
|
||||
raise StorageError("boto3 not installed. Install with: pip install boto3")
|
||||
|
||||
def save_file(self, file_hash: str, content: bytes, filename: Optional[str] = None) -> str:
|
||||
"""Upload file to S3"""
|
||||
try:
|
||||
# Get storage key
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Upload to S3
|
||||
self.s3_client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentType='application/octet-stream',
|
||||
Metadata={
|
||||
'original_filename': filename or '',
|
||||
'file_hash': file_hash
|
||||
}
|
||||
)
|
||||
|
||||
logger.debug(f"File uploaded to S3: {key} ({len(content)} bytes)")
|
||||
|
||||
return key
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to upload file to S3 {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to upload file: {e}")
|
||||
|
||||
def get_file(self, file_hash: str) -> bytes:
|
||||
"""Download file from S3"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Download from S3
|
||||
response = self.s3_client.get_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key
|
||||
)
|
||||
|
||||
content = response['Body'].read()
|
||||
|
||||
logger.debug(f"File downloaded from S3: {key} ({len(content)} bytes)")
|
||||
|
||||
return content
|
||||
|
||||
except self.ClientError as e:
|
||||
if e.response['Error']['Code'] == 'NoSuchKey':
|
||||
raise FileNotFoundError(f"File not found: {file_hash}")
|
||||
logger.error(f"Failed to download file from S3 {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to download file: {e}")
|
||||
|
||||
def file_exists(self, file_hash: str) -> bool:
|
||||
"""Check if file exists in S3"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
self.s3_client.head_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
except self.ClientError as e:
|
||||
if e.response['Error']['Code'] == '404':
|
||||
return False
|
||||
logger.error(f"Failed to check file existence in S3 {file_hash}: {e}")
|
||||
return False
|
||||
|
||||
def delete_file(self, file_hash: str) -> bool:
|
||||
"""Delete file from S3"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Check if file exists first
|
||||
if not self.file_exists(file_hash):
|
||||
return False
|
||||
|
||||
# Delete from S3
|
||||
self.s3_client.delete_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key
|
||||
)
|
||||
|
||||
logger.debug(f"File deleted from S3: {key}")
|
||||
|
||||
return True
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to delete file from S3 {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to delete file: {e}")
|
||||
|
||||
def get_file_url(self, file_hash: str, expires_in: int = 3600) -> str:
|
||||
"""
|
||||
Get presigned URL for file access
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
expires_in: URL expiration time in seconds (default: 1 hour)
|
||||
|
||||
Returns:
|
||||
Presigned URL for file access
|
||||
"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Generate presigned URL
|
||||
url = self.s3_client.generate_presigned_url(
|
||||
'get_object',
|
||||
Params={
|
||||
'Bucket': self.bucket,
|
||||
'Key': key
|
||||
},
|
||||
ExpiresIn=expires_in
|
||||
)
|
||||
|
||||
return url
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to generate presigned URL for {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to generate URL: {e}")
|
||||
|
||||
def get_storage_stats(self) -> dict:
|
||||
"""
|
||||
Get storage statistics from S3
|
||||
|
||||
Returns:
|
||||
Dictionary with storage stats
|
||||
"""
|
||||
try:
|
||||
# List all objects with .sub extension
|
||||
paginator = self.s3_client.get_paginator('list_objects_v2')
|
||||
pages = paginator.paginate(Bucket=self.bucket, Prefix='')
|
||||
|
||||
total_files = 0
|
||||
total_size = 0
|
||||
|
||||
for page in pages:
|
||||
if 'Contents' in page:
|
||||
for obj in page['Contents']:
|
||||
if obj['Key'].endswith('.sub'):
|
||||
total_files += 1
|
||||
total_size += obj['Size']
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"bucket": self.bucket,
|
||||
"region": self.region
|
||||
}
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to get S3 storage stats: {e}")
|
||||
return {
|
||||
"error": str(e)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
GigLez Database Package
|
||||
|
||||
SQLAlchemy ORM models and database utilities
|
||||
"""
|
||||
|
||||
from .models import (
|
||||
Base,
|
||||
User,
|
||||
Session,
|
||||
Device,
|
||||
Capture,
|
||||
Signature,
|
||||
CaptureMatch,
|
||||
Identification,
|
||||
Vote,
|
||||
UploadMarker,
|
||||
FlipperSignature,
|
||||
RTL433Protocol
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'Base',
|
||||
'User',
|
||||
'Session',
|
||||
'Device',
|
||||
'Capture',
|
||||
'Signature',
|
||||
'CaptureMatch',
|
||||
'Identification',
|
||||
'Vote',
|
||||
'UploadMarker',
|
||||
'FlipperSignature',
|
||||
'RTL433Protocol'
|
||||
]
|
||||
@@ -0,0 +1,564 @@
|
||||
"""
|
||||
GigLez SQLAlchemy ORM Models
|
||||
|
||||
Database models matching the PostgreSQL + PostGIS schema
|
||||
Based on Wigle wardriving patterns adapted for IoT RF device mapping
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
from sqlalchemy import (
|
||||
Column, String, Integer, Float, DateTime, Text, Boolean,
|
||||
DECIMAL, ARRAY, ForeignKey, CheckConstraint, UniqueConstraint,
|
||||
Index, BYTEA
|
||||
)
|
||||
from sqlalchemy.orm import declarative_base, relationship
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from geoalchemy2 import Geometry
|
||||
from geoalchemy2.functions import ST_SetSRID, ST_MakePoint
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BASE
|
||||
# =============================================================================
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CORE MODELS
|
||||
# =============================================================================
|
||||
|
||||
class User(Base):
|
||||
"""User accounts for community features"""
|
||||
|
||||
__tablename__ = 'users'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Authentication
|
||||
username = Column(String(50), unique=True, nullable=False, index=True)
|
||||
email = Column(String(255), unique=True, nullable=False, index=True)
|
||||
password_hash = Column(String(255), nullable=False)
|
||||
|
||||
# Profile
|
||||
display_name = Column(String(100))
|
||||
avatar_url = Column(Text)
|
||||
bio = Column(Text)
|
||||
|
||||
# Statistics
|
||||
total_captures = Column(Integer, default=0)
|
||||
total_identifications = Column(Integer, default=0)
|
||||
reputation_score = Column(Integer, default=0)
|
||||
|
||||
# Settings
|
||||
api_key = Column(String(64), unique=True, index=True)
|
||||
email_verified = Column(Boolean, default=False)
|
||||
|
||||
# Timestamps
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
last_login = Column(DateTime)
|
||||
|
||||
# Relationships
|
||||
sessions = relationship("Session", back_populates="user")
|
||||
captures = relationship("Capture", back_populates="user")
|
||||
identifications = relationship("Identification", back_populates="user")
|
||||
votes = relationship("Vote", back_populates="user")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<User(id={self.id}, username='{self.username}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'username': self.username,
|
||||
'display_name': self.display_name,
|
||||
'total_captures': self.total_captures,
|
||||
'reputation_score': self.reputation_score,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None
|
||||
}
|
||||
|
||||
|
||||
class Session(Base):
|
||||
"""Wardriving/capture sessions (Wigle pattern)"""
|
||||
|
||||
__tablename__ = 'sessions'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
user_id = Column(Integer, ForeignKey('users.id', ondelete='SET NULL'), index=True)
|
||||
|
||||
# Session Metadata
|
||||
session_uuid = Column(String(64), unique=True, nullable=False, index=True)
|
||||
name = Column(String(200))
|
||||
description = Column(Text)
|
||||
started_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
|
||||
ended_at = Column(DateTime)
|
||||
|
||||
# Privacy Settings
|
||||
is_public = Column(Boolean, default=True, index=True)
|
||||
anonymize_gps = Column(Boolean, default=False)
|
||||
gps_precision_meters = Column(Integer, default=10)
|
||||
|
||||
# Session Statistics (auto-updated by triggers)
|
||||
total_captures = Column(Integer, default=0)
|
||||
unique_devices = Column(Integer, default=0)
|
||||
distance_km = Column(DECIMAL(10, 2))
|
||||
|
||||
# Bounding Box (auto-updated by triggers)
|
||||
min_latitude = Column(DECIMAL(10, 8))
|
||||
max_latitude = Column(DECIMAL(10, 8))
|
||||
min_longitude = Column(DECIMAL(11, 8))
|
||||
max_longitude = Column(DECIMAL(11, 8))
|
||||
|
||||
# Relationships
|
||||
user = relationship("User", back_populates="sessions")
|
||||
captures = relationship("Capture", back_populates="session")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Session(id={self.id}, uuid='{self.session_uuid}', name='{self.name}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'session_uuid': self.session_uuid,
|
||||
'name': self.name,
|
||||
'started_at': self.started_at.isoformat() if self.started_at else None,
|
||||
'ended_at': self.ended_at.isoformat() if self.ended_at else None,
|
||||
'total_captures': self.total_captures,
|
||||
'unique_devices': self.unique_devices,
|
||||
'is_public': self.is_public
|
||||
}
|
||||
|
||||
|
||||
class Device(Base):
|
||||
"""Known IoT device types from signature databases"""
|
||||
|
||||
__tablename__ = 'devices'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Device Identification
|
||||
manufacturer = Column(String(200), index=True)
|
||||
model = Column(String(200))
|
||||
device_type = Column(String(100), index=True)
|
||||
description = Column(Text)
|
||||
|
||||
# RF Characteristics
|
||||
typical_frequency = Column(Integer, index=True)
|
||||
frequency_range_low = Column(Integer)
|
||||
frequency_range_high = Column(Integer)
|
||||
modulation_types = Column(ARRAY(Text))
|
||||
|
||||
# Protocol Information
|
||||
protocol = Column(String(100), index=True)
|
||||
bit_length = Column(Integer)
|
||||
encoding = Column(String(50))
|
||||
|
||||
# Metadata
|
||||
fcc_id = Column(String(50))
|
||||
manufacturer_code = Column(String(50))
|
||||
|
||||
# Community Data
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
verified = Column(Boolean, default=False, index=True)
|
||||
verification_count = Column(Integer, default=0)
|
||||
|
||||
# Source
|
||||
source = Column(String(50), index=True) # 'flipper', 'rtl433', 'urh', 'community'
|
||||
source_url = Column(Text)
|
||||
|
||||
# Full-text search (auto-updated by trigger)
|
||||
search_vector = Column('search_vector', nullable=True)
|
||||
|
||||
# Relationships
|
||||
signatures = relationship("Signature", back_populates="device")
|
||||
captures = relationship("Capture", back_populates="device")
|
||||
capture_matches = relationship("CaptureMatch", back_populates="device")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Device(id={self.id}, manufacturer='{self.manufacturer}', model='{self.model}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'manufacturer': self.manufacturer,
|
||||
'model': self.model,
|
||||
'device_type': self.device_type,
|
||||
'description': self.description,
|
||||
'typical_frequency': self.typical_frequency,
|
||||
'protocol': self.protocol,
|
||||
'verified': self.verified,
|
||||
'source': self.source
|
||||
}
|
||||
|
||||
|
||||
class Capture(Base):
|
||||
"""RF signal captures with GPS coordinates"""
|
||||
|
||||
__tablename__ = 'captures'
|
||||
|
||||
# Primary Key: SHA256 hash of file (Wigle deduplication pattern)
|
||||
file_hash = Column(String(64), primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
session_id = Column(Integer, ForeignKey('sessions.id', ondelete='CASCADE'), index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id', ondelete='SET NULL'), index=True)
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
|
||||
|
||||
# GPS Data
|
||||
latitude = Column(DECIMAL(10, 8), nullable=False)
|
||||
longitude = Column(DECIMAL(11, 8), nullable=False)
|
||||
altitude = Column(DECIMAL(8, 2))
|
||||
gps_accuracy = Column(DECIMAL(6, 2))
|
||||
geom = Column(Geometry('POINT', srid=4326)) # Auto-populated by trigger
|
||||
|
||||
# Timestamps
|
||||
captured_at = Column(DateTime, nullable=False, index=True)
|
||||
uploaded_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
|
||||
# RF Signal Data
|
||||
frequency = Column(Integer, nullable=False, index=True)
|
||||
rssi = Column(Integer)
|
||||
modulation = Column(String(50))
|
||||
preset = Column(String(100))
|
||||
|
||||
# Protocol Information (if decoded)
|
||||
protocol = Column(String(100), index=True)
|
||||
bit_length = Column(Integer)
|
||||
key_data = Column(BYTEA)
|
||||
timing_element = Column(Integer)
|
||||
|
||||
# Raw Signal Data
|
||||
raw_data = Column(Text)
|
||||
raw_format = Column(String(20)) # 'RAW', 'BinRAW', 'KEY'
|
||||
|
||||
# File Storage
|
||||
file_path = Column(String(500))
|
||||
file_size = Column(Integer)
|
||||
|
||||
# Automatic Matching Results (best match)
|
||||
match_confidence = Column(DECIMAL(5, 4))
|
||||
match_method = Column(String(50))
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
CheckConstraint('latitude >= -90 AND latitude <= 90', name='valid_latitude'),
|
||||
CheckConstraint('longitude >= -180 AND longitude <= 180', name='valid_longitude'),
|
||||
CheckConstraint('match_confidence >= 0 AND match_confidence <= 1', name='valid_confidence'),
|
||||
CheckConstraint('frequency >= 300000000 AND frequency <= 928000000', name='valid_frequency'),
|
||||
Index('idx_captures_geom', 'geom', postgresql_using='gist'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
session = relationship("Session", back_populates="captures")
|
||||
user = relationship("User", back_populates="captures")
|
||||
device = relationship("Device", back_populates="captures")
|
||||
capture_matches = relationship("CaptureMatch", back_populates="capture")
|
||||
identifications = relationship("Identification", back_populates="capture")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Capture(hash='{self.file_hash[:8]}...', freq={self.frequency}, protocol='{self.protocol}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'file_hash': self.file_hash,
|
||||
'latitude': float(self.latitude) if self.latitude else None,
|
||||
'longitude': float(self.longitude) if self.longitude else None,
|
||||
'altitude': float(self.altitude) if self.altitude else None,
|
||||
'gps_accuracy': float(self.gps_accuracy) if self.gps_accuracy else None,
|
||||
'captured_at': self.captured_at.isoformat() if self.captured_at else None,
|
||||
'frequency': self.frequency,
|
||||
'rssi': self.rssi,
|
||||
'modulation': self.modulation,
|
||||
'protocol': self.protocol,
|
||||
'bit_length': self.bit_length,
|
||||
'device_id': self.device_id,
|
||||
'match_confidence': float(self.match_confidence) if self.match_confidence else None,
|
||||
'match_method': self.match_method
|
||||
}
|
||||
|
||||
|
||||
class Signature(Base):
|
||||
"""Protocol signatures for device matching"""
|
||||
|
||||
__tablename__ = 'signatures'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='CASCADE'), index=True)
|
||||
|
||||
# Signature Pattern
|
||||
protocol = Column(String(100), nullable=False, index=True)
|
||||
frequency = Column(Integer, index=True)
|
||||
modulation = Column(String(50))
|
||||
|
||||
# Matching Criteria
|
||||
bit_pattern = Column(BYTEA)
|
||||
bit_mask = Column(BYTEA)
|
||||
timing_min = Column(Integer)
|
||||
timing_max = Column(Integer)
|
||||
|
||||
# Raw Pattern
|
||||
pattern_regex = Column(Text)
|
||||
|
||||
# Confidence Weighting
|
||||
weight = Column(DECIMAL(4, 3), default=1.0)
|
||||
|
||||
# Metadata
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
source = Column(String(50), index=True)
|
||||
source_file = Column(String(500))
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
UniqueConstraint('device_id', 'protocol', 'bit_pattern', name='uq_device_protocol_pattern'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
device = relationship("Device", back_populates="signatures")
|
||||
flipper_signature = relationship("FlipperSignature", back_populates="signature", uselist=False)
|
||||
rtl433_protocol = relationship("RTL433Protocol", back_populates="signature", uselist=False)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Signature(id={self.id}, protocol='{self.protocol}', device_id={self.device_id})>"
|
||||
|
||||
|
||||
class CaptureMatch(Base):
|
||||
"""Many-to-many mapping of captures to devices with confidence scores"""
|
||||
|
||||
__tablename__ = 'capture_matches'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
capture_id = Column(String(64), ForeignKey('captures.file_hash', ondelete='CASCADE'), index=True)
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='CASCADE'), index=True)
|
||||
signature_id = Column(Integer, ForeignKey('signatures.id', ondelete='SET NULL'), index=True)
|
||||
|
||||
# Match Details
|
||||
confidence = Column(DECIMAL(5, 4), nullable=False, index=True)
|
||||
match_method = Column(String(50), nullable=False)
|
||||
match_details = Column(JSONB)
|
||||
|
||||
# Timestamp
|
||||
matched_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
CheckConstraint('confidence >= 0 AND confidence <= 1', name='valid_match_confidence'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
capture = relationship("Capture", back_populates="capture_matches")
|
||||
device = relationship("Device", back_populates="capture_matches")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<CaptureMatch(capture='{self.capture_id[:8]}...', device={self.device_id}, confidence={self.confidence})>"
|
||||
|
||||
|
||||
class Identification(Base):
|
||||
"""User-submitted device identifications"""
|
||||
|
||||
__tablename__ = 'identifications'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
capture_id = Column(String(64), ForeignKey('captures.file_hash', ondelete='CASCADE'), index=True)
|
||||
device_id = Column(Integer, ForeignKey('devices.id'), index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id'), index=True)
|
||||
|
||||
# Identification Details
|
||||
confidence = Column(String(20)) # 'certain', 'likely', 'guess'
|
||||
notes = Column(Text)
|
||||
|
||||
# Visual Evidence
|
||||
photo_urls = Column(ARRAY(Text))
|
||||
|
||||
# Community Validation
|
||||
upvotes = Column(Integer, default=0)
|
||||
downvotes = Column(Integer, default=0)
|
||||
verified = Column(Boolean, default=False, index=True)
|
||||
verified_by = Column(Integer, ForeignKey('users.id'))
|
||||
verified_at = Column(DateTime)
|
||||
|
||||
# Timestamp
|
||||
submitted_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
UniqueConstraint('capture_id', 'user_id', 'device_id', name='uq_capture_user_device'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
capture = relationship("Capture", back_populates="identifications")
|
||||
user = relationship("User", back_populates="identifications", foreign_keys=[user_id])
|
||||
votes = relationship("Vote", back_populates="identification")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Identification(id={self.id}, capture='{self.capture_id[:8]}...', device={self.device_id})>"
|
||||
|
||||
|
||||
class Vote(Base):
|
||||
"""Votes on device identifications"""
|
||||
|
||||
__tablename__ = 'votes'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
identification_id = Column(Integer, ForeignKey('identifications.id', ondelete='CASCADE'), index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id'), index=True)
|
||||
|
||||
# Vote Data
|
||||
vote_type = Column(Integer, nullable=False) # 1 = upvote, -1 = downvote
|
||||
voted_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
CheckConstraint('vote_type IN (1, -1)', name='valid_vote'),
|
||||
UniqueConstraint('identification_id', 'user_id', name='uq_identification_user'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
identification = relationship("Identification", back_populates="votes")
|
||||
user = relationship("User", back_populates="votes")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Vote(id={self.id}, identification={self.identification_id}, type={self.vote_type})>"
|
||||
|
||||
|
||||
class UploadMarker(Base):
|
||||
"""Track last uploaded capture per user/session (Wigle incremental sync pattern)"""
|
||||
|
||||
__tablename__ = 'upload_markers'
|
||||
|
||||
# Composite Primary Key
|
||||
user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
|
||||
session_id = Column(Integer, ForeignKey('sessions.id', ondelete='CASCADE'), primary_key=True)
|
||||
|
||||
# Marker Data
|
||||
last_capture_hash = Column(String(64))
|
||||
last_upload_time = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<UploadMarker(user={self.user_id}, session={self.session_id})>"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# SIGNATURE DATABASE MODELS
|
||||
# =============================================================================
|
||||
|
||||
class FlipperSignature(Base):
|
||||
"""Flipper Zero specific signature data"""
|
||||
|
||||
__tablename__ = 'flipper_signatures'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Key
|
||||
signature_id = Column(Integer, ForeignKey('signatures.id', ondelete='CASCADE'), index=True)
|
||||
|
||||
# Flipper-Specific Fields
|
||||
filetype = Column(String(50))
|
||||
version = Column(Integer)
|
||||
preset = Column(String(100), index=True)
|
||||
|
||||
# Custom Preset Data
|
||||
custom_preset_module = Column(String(50))
|
||||
custom_preset_data = Column(BYTEA)
|
||||
|
||||
# Protocol Data
|
||||
protocol = Column(String(100), index=True)
|
||||
bit = Column(Integer)
|
||||
key = Column(BYTEA)
|
||||
te = Column(Integer)
|
||||
|
||||
# RAW Data
|
||||
raw_data = Column(Text)
|
||||
bin_raw_bit = Column(Integer)
|
||||
bin_raw_te = Column(Integer)
|
||||
bin_raw_data = Column(BYTEA)
|
||||
|
||||
# Source
|
||||
source_file = Column(String(500))
|
||||
imported_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
signature = relationship("Signature", back_populates="flipper_signature")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<FlipperSignature(id={self.id}, protocol='{self.protocol}')>"
|
||||
|
||||
|
||||
class RTL433Protocol(Base):
|
||||
"""RTL_433 specific protocol data"""
|
||||
|
||||
__tablename__ = 'rtl433_protocols'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Key
|
||||
signature_id = Column(Integer, ForeignKey('signatures.id', ondelete='CASCADE'), index=True)
|
||||
|
||||
# Protocol Identification
|
||||
protocol_number = Column(Integer, index=True)
|
||||
protocol_name = Column(String(200))
|
||||
model = Column(String(200), index=True)
|
||||
|
||||
# RF Characteristics
|
||||
frequency = Column(Integer)
|
||||
modulation = Column(String(50), index=True)
|
||||
|
||||
# Timing Information
|
||||
short_width = Column(Integer)
|
||||
long_width = Column(Integer)
|
||||
reset_limit = Column(Integer)
|
||||
gap_limit = Column(Integer)
|
||||
|
||||
# Decoding
|
||||
decoder_type = Column(String(50))
|
||||
bit_count = Column(Integer)
|
||||
|
||||
# JSON Fields Mapping
|
||||
json_fields = Column(JSONB)
|
||||
|
||||
# Source
|
||||
source_file = Column(String(500))
|
||||
imported_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
signature = relationship("Signature", back_populates="rtl433_protocol")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<RTL433Protocol(id={self.id}, name='{self.protocol_name}')>"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# HELPER FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
def create_all_tables(engine):
|
||||
"""Create all tables in the database"""
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
def drop_all_tables(engine):
|
||||
"""Drop all tables from the database (use with caution!)"""
|
||||
Base.metadata.drop_all(engine)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
GigLez GPS Package
|
||||
|
||||
GPS validation and utilities
|
||||
"""
|
||||
|
||||
from .validator import (
|
||||
validate_gps_coordinates,
|
||||
is_null_island,
|
||||
is_valid_accuracy,
|
||||
GPSValidator,
|
||||
GPSCoordinate
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'validate_gps_coordinates',
|
||||
'is_null_island',
|
||||
'is_valid_accuracy',
|
||||
'GPSValidator',
|
||||
'GPSCoordinate'
|
||||
]
|
||||
@@ -0,0 +1,464 @@
|
||||
"""
|
||||
GPS Coordinate Validation
|
||||
|
||||
Based on Wigle Android client GPS validation patterns
|
||||
See docs/wigle_analysis.md Section 3 for details
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Tuple
|
||||
from datetime import datetime
|
||||
from loguru import logger
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# GPS VALIDATION THRESHOLDS (Wigle-derived)
|
||||
# =============================================================================
|
||||
|
||||
# Maximum GPS accuracy threshold (meters)
|
||||
# Wigle uses 32m, we use 50m for broader acceptance
|
||||
MAX_GPS_ACCURACY = 50.0
|
||||
|
||||
# Minimum GPS accuracy for high-quality captures
|
||||
MIN_GPS_ACCURACY = 10.0
|
||||
|
||||
# Null Island detection threshold (degrees)
|
||||
# Coordinates within this distance of (0,0) are rejected
|
||||
NULL_ISLAND_THRESHOLD = 0.001
|
||||
|
||||
# Valid coordinate ranges
|
||||
MIN_LATITUDE = -90.0
|
||||
MAX_LATITUDE = 90.0
|
||||
MIN_LONGITUDE = -180.0
|
||||
MAX_LONGITUDE = 180.0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DATA CLASSES
|
||||
# =============================================================================
|
||||
|
||||
@dataclass
|
||||
class GPSCoordinate:
|
||||
"""GPS coordinate with metadata"""
|
||||
|
||||
latitude: float
|
||||
longitude: float
|
||||
altitude: Optional[float] = None
|
||||
accuracy: Optional[float] = None
|
||||
timestamp: Optional[datetime] = None
|
||||
provider: Optional[str] = None # 'gps', 'network', 'fused'
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate after initialization"""
|
||||
if not validate_gps_coordinates(self.latitude, self.longitude, self.accuracy):
|
||||
raise ValueError(
|
||||
f"Invalid GPS coordinates: lat={self.latitude}, lon={self.longitude}, "
|
||||
f"accuracy={self.accuracy}"
|
||||
)
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'latitude': self.latitude,
|
||||
'longitude': self.longitude,
|
||||
'altitude': self.altitude,
|
||||
'accuracy': self.accuracy,
|
||||
'timestamp': self.timestamp.isoformat() if self.timestamp else None,
|
||||
'provider': self.provider
|
||||
}
|
||||
|
||||
def is_high_quality(self) -> bool:
|
||||
"""Check if this is a high-quality GPS fix"""
|
||||
if self.accuracy is None:
|
||||
return False
|
||||
return self.accuracy <= MIN_GPS_ACCURACY
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# VALIDATION FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
def validate_gps_coordinates(
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
accuracy: Optional[float] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Validate GPS coordinates using Wigle patterns
|
||||
|
||||
Checks:
|
||||
1. Valid coordinate ranges (-90 to 90, -180 to 180)
|
||||
2. Null Island detection (reject 0.0, 0.0)
|
||||
3. Accuracy threshold (< 50 meters)
|
||||
|
||||
Args:
|
||||
latitude: Latitude in decimal degrees
|
||||
longitude: Longitude in decimal degrees
|
||||
accuracy: GPS accuracy in meters (optional)
|
||||
|
||||
Returns:
|
||||
True if valid, False otherwise
|
||||
|
||||
Example:
|
||||
>>> validate_gps_coordinates(40.7128, -74.0060, 5.0)
|
||||
True
|
||||
>>> validate_gps_coordinates(0.0, 0.0)
|
||||
False
|
||||
>>> validate_gps_coordinates(40.7128, -74.0060, 100.0)
|
||||
False
|
||||
"""
|
||||
|
||||
# Check coordinate ranges
|
||||
if not (MIN_LATITUDE <= latitude <= MAX_LATITUDE):
|
||||
logger.warning(f"Latitude out of range: {latitude}")
|
||||
return False
|
||||
|
||||
if not (MIN_LONGITUDE <= longitude <= MAX_LONGITUDE):
|
||||
logger.warning(f"Longitude out of range: {longitude}")
|
||||
return False
|
||||
|
||||
# Check for Null Island (0, 0)
|
||||
if is_null_island(latitude, longitude):
|
||||
logger.warning(f"Null Island detected: ({latitude}, {longitude})")
|
||||
return False
|
||||
|
||||
# Check accuracy threshold
|
||||
if accuracy is not None and not is_valid_accuracy(accuracy):
|
||||
logger.warning(f"GPS accuracy too low: {accuracy}m")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_null_island(latitude: float, longitude: float) -> bool:
|
||||
"""
|
||||
Check if coordinates are at Null Island (0, 0)
|
||||
|
||||
Null Island is the point where the prime meridian and equator intersect.
|
||||
GPS errors often result in (0.0, 0.0) coordinates, so we reject these.
|
||||
|
||||
Args:
|
||||
latitude: Latitude in decimal degrees
|
||||
longitude: Longitude in decimal degrees
|
||||
|
||||
Returns:
|
||||
True if coordinates are near (0, 0), False otherwise
|
||||
"""
|
||||
return (abs(latitude) < NULL_ISLAND_THRESHOLD and
|
||||
abs(longitude) < NULL_ISLAND_THRESHOLD)
|
||||
|
||||
|
||||
def is_valid_accuracy(accuracy: float) -> bool:
|
||||
"""
|
||||
Check if GPS accuracy is within acceptable threshold
|
||||
|
||||
Wigle uses 32m threshold. We use 50m for broader acceptance.
|
||||
|
||||
Args:
|
||||
accuracy: GPS accuracy in meters
|
||||
|
||||
Returns:
|
||||
True if accuracy is acceptable, False otherwise
|
||||
"""
|
||||
return 0 < accuracy <= MAX_GPS_ACCURACY
|
||||
|
||||
|
||||
def check_coordinate_bounds(latitude: float, longitude: float) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Check if coordinates are within valid bounds
|
||||
|
||||
Args:
|
||||
latitude: Latitude in decimal degrees
|
||||
longitude: Longitude in decimal degrees
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_message)
|
||||
"""
|
||||
if not (MIN_LATITUDE <= latitude <= MAX_LATITUDE):
|
||||
return False, f"Latitude {latitude} out of range [{MIN_LATITUDE}, {MAX_LATITUDE}]"
|
||||
|
||||
if not (MIN_LONGITUDE <= longitude <= MAX_LONGITUDE):
|
||||
return False, f"Longitude {longitude} out of range [{MIN_LONGITUDE}, {MAX_LONGITUDE}]"
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def anonymize_gps(
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
precision_meters: int = 100
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Anonymize GPS coordinates by reducing precision
|
||||
|
||||
Wigle pattern: Allow users to control GPS precision for privacy
|
||||
|
||||
Args:
|
||||
latitude: Original latitude
|
||||
longitude: Original longitude
|
||||
precision_meters: Desired precision in meters (10, 100, 1000)
|
||||
|
||||
Returns:
|
||||
Tuple of (anonymized_lat, anonymized_lon)
|
||||
|
||||
Example:
|
||||
>>> anonymize_gps(40.7128456, -74.0059728, 100)
|
||||
(40.71, -74.01)
|
||||
"""
|
||||
# Rough conversion: 1 degree ≈ 111 km
|
||||
# 100m precision ≈ 0.001 degrees
|
||||
# 1000m precision ≈ 0.01 degrees
|
||||
|
||||
if precision_meters <= 10:
|
||||
decimals = 5 # ~1.1m
|
||||
elif precision_meters <= 100:
|
||||
decimals = 3 # ~111m
|
||||
elif precision_meters <= 1000:
|
||||
decimals = 2 # ~1.1km
|
||||
else:
|
||||
decimals = 1 # ~11km
|
||||
|
||||
lat_anon = round(latitude, decimals)
|
||||
lon_anon = round(longitude, decimals)
|
||||
|
||||
return lat_anon, lon_anon
|
||||
|
||||
|
||||
def calculate_distance(
|
||||
lat1: float, lon1: float,
|
||||
lat2: float, lon2: float
|
||||
) -> float:
|
||||
"""
|
||||
Calculate distance between two GPS coordinates using Haversine formula
|
||||
|
||||
Args:
|
||||
lat1: First latitude
|
||||
lon1: First longitude
|
||||
lat2: Second latitude
|
||||
lon2: Second longitude
|
||||
|
||||
Returns:
|
||||
Distance in kilometers
|
||||
|
||||
Example:
|
||||
>>> calculate_distance(40.7128, -74.0060, 40.7614, -73.9776)
|
||||
8.67 # ~8.67 km from downtown Manhattan to Central Park
|
||||
"""
|
||||
from math import radians, sin, cos, sqrt, atan2
|
||||
|
||||
R = 6371.0 # Earth radius in kilometers
|
||||
|
||||
lat1_rad = radians(lat1)
|
||||
lon1_rad = radians(lon1)
|
||||
lat2_rad = radians(lat2)
|
||||
lon2_rad = radians(lon2)
|
||||
|
||||
dlat = lat2_rad - lat1_rad
|
||||
dlon = lon2_rad - lon1_rad
|
||||
|
||||
a = sin(dlat / 2)**2 + cos(lat1_rad) * cos(lat2_rad) * sin(dlon / 2)**2
|
||||
c = 2 * atan2(sqrt(a), sqrt(1 - a))
|
||||
|
||||
distance = R * c
|
||||
return distance
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# GPS VALIDATOR CLASS
|
||||
# =============================================================================
|
||||
|
||||
class GPSValidator:
|
||||
"""
|
||||
GPS validator with configurable thresholds
|
||||
|
||||
Based on Wigle's multi-level validation approach
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_accuracy: float = MAX_GPS_ACCURACY,
|
||||
min_accuracy: float = MIN_GPS_ACCURACY,
|
||||
allow_null_island: bool = False,
|
||||
strict_mode: bool = False
|
||||
):
|
||||
"""
|
||||
Initialize GPS validator
|
||||
|
||||
Args:
|
||||
max_accuracy: Maximum acceptable accuracy (meters)
|
||||
min_accuracy: Minimum accuracy for high-quality captures
|
||||
allow_null_island: Allow (0, 0) coordinates (not recommended)
|
||||
strict_mode: Reject coordinates without accuracy data
|
||||
"""
|
||||
self.max_accuracy = max_accuracy
|
||||
self.min_accuracy = min_accuracy
|
||||
self.allow_null_island = allow_null_island
|
||||
self.strict_mode = strict_mode
|
||||
|
||||
# Statistics
|
||||
self.total_validated = 0
|
||||
self.total_accepted = 0
|
||||
self.total_rejected = 0
|
||||
self.rejection_reasons = {}
|
||||
|
||||
def validate(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
accuracy: Optional[float] = None
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Validate GPS coordinates
|
||||
|
||||
Args:
|
||||
latitude: Latitude in decimal degrees
|
||||
longitude: Longitude in decimal degrees
|
||||
accuracy: GPS accuracy in meters (optional)
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, rejection_reason)
|
||||
"""
|
||||
self.total_validated += 1
|
||||
|
||||
# Check bounds
|
||||
is_valid, error = check_coordinate_bounds(latitude, longitude)
|
||||
if not is_valid:
|
||||
self.total_rejected += 1
|
||||
self._record_rejection("out_of_bounds")
|
||||
return False, error
|
||||
|
||||
# Check Null Island (unless allowed)
|
||||
if not self.allow_null_island and is_null_island(latitude, longitude):
|
||||
self.total_rejected += 1
|
||||
self._record_rejection("null_island")
|
||||
return False, "Null Island (0.0, 0.0) detected"
|
||||
|
||||
# Check accuracy
|
||||
if self.strict_mode and accuracy is None:
|
||||
self.total_rejected += 1
|
||||
self._record_rejection("missing_accuracy")
|
||||
return False, "Missing accuracy data (strict mode)"
|
||||
|
||||
if accuracy is not None and not (0 < accuracy <= self.max_accuracy):
|
||||
self.total_rejected += 1
|
||||
self._record_rejection("poor_accuracy")
|
||||
return False, f"GPS accuracy {accuracy}m exceeds threshold {self.max_accuracy}m"
|
||||
|
||||
# All checks passed
|
||||
self.total_accepted += 1
|
||||
return True, None
|
||||
|
||||
def validate_coordinate(self, coord: GPSCoordinate) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Validate a GPSCoordinate object
|
||||
|
||||
Args:
|
||||
coord: GPSCoordinate object
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, rejection_reason)
|
||||
"""
|
||||
return self.validate(coord.latitude, coord.longitude, coord.accuracy)
|
||||
|
||||
def _record_rejection(self, reason: str):
|
||||
"""Record rejection reason for statistics"""
|
||||
self.rejection_reasons[reason] = self.rejection_reasons.get(reason, 0) + 1
|
||||
|
||||
def get_statistics(self) -> dict:
|
||||
"""Get validation statistics"""
|
||||
return {
|
||||
'total_validated': self.total_validated,
|
||||
'total_accepted': self.total_accepted,
|
||||
'total_rejected': self.total_rejected,
|
||||
'acceptance_rate': (
|
||||
self.total_accepted / self.total_validated
|
||||
if self.total_validated > 0 else 0.0
|
||||
),
|
||||
'rejection_reasons': self.rejection_reasons
|
||||
}
|
||||
|
||||
def reset_statistics(self):
|
||||
"""Reset validation statistics"""
|
||||
self.total_validated = 0
|
||||
self.total_accepted = 0
|
||||
self.total_rejected = 0
|
||||
self.rejection_reasons = {}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CONVENIENCE FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
def create_gps_coordinate(
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
altitude: Optional[float] = None,
|
||||
accuracy: Optional[float] = None,
|
||||
timestamp: Optional[datetime] = None,
|
||||
provider: Optional[str] = None
|
||||
) -> Optional[GPSCoordinate]:
|
||||
"""
|
||||
Create a validated GPSCoordinate
|
||||
|
||||
Args:
|
||||
latitude: Latitude in decimal degrees
|
||||
longitude: Longitude in decimal degrees
|
||||
altitude: Altitude in meters (optional)
|
||||
accuracy: GPS accuracy in meters (optional)
|
||||
timestamp: Capture timestamp (optional)
|
||||
provider: GPS provider (optional)
|
||||
|
||||
Returns:
|
||||
GPSCoordinate if valid, None otherwise
|
||||
"""
|
||||
try:
|
||||
return GPSCoordinate(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
altitude=altitude,
|
||||
accuracy=accuracy,
|
||||
timestamp=timestamp,
|
||||
provider=provider
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.warning(f"Failed to create GPS coordinate: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TESTING
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test validation
|
||||
print("GPS Validator Tests")
|
||||
print("=" * 60)
|
||||
|
||||
# Test cases
|
||||
test_cases = [
|
||||
# (lat, lon, accuracy, expected_valid, description)
|
||||
(40.7128, -74.0060, 5.0, True, "NYC - high quality"),
|
||||
(40.7128, -74.0060, 25.0, True, "NYC - acceptable quality"),
|
||||
(40.7128, -74.0060, 100.0, False, "NYC - poor accuracy"),
|
||||
(0.0, 0.0, 5.0, False, "Null Island"),
|
||||
(91.0, 0.0, 5.0, False, "Latitude out of range"),
|
||||
(0.0, 181.0, 5.0, False, "Longitude out of range"),
|
||||
(51.5074, -0.1278, 8.0, True, "London - high quality"),
|
||||
]
|
||||
|
||||
validator = GPSValidator()
|
||||
|
||||
for lat, lon, acc, expected, desc in test_cases:
|
||||
is_valid, reason = validator.validate(lat, lon, acc)
|
||||
status = "✅" if is_valid == expected else "❌"
|
||||
print(f"{status} {desc}")
|
||||
print(f" lat={lat}, lon={lon}, accuracy={acc}m")
|
||||
print(f" Valid: {is_valid}, Reason: {reason}")
|
||||
print()
|
||||
|
||||
# Print statistics
|
||||
print("Validation Statistics:")
|
||||
print("=" * 60)
|
||||
stats = validator.get_statistics()
|
||||
for key, value in stats.items():
|
||||
print(f"{key}: {value}")
|
||||
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
Device signature matching engine
|
||||
"""
|
||||
|
||||
from .engine import SignatureMatcher, MatchResult
|
||||
from .strategies import ExactMatcher, PartialMatcher, PatternMatcher, TimingMatcher
|
||||
|
||||
__all__ = [
|
||||
'SignatureMatcher',
|
||||
'MatchResult',
|
||||
'ExactMatcher',
|
||||
'PartialMatcher',
|
||||
'PatternMatcher',
|
||||
'TimingMatcher'
|
||||
]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""
|
||||
Main signature matching engine
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Dict, Any
|
||||
from loguru import logger
|
||||
|
||||
from ..parser.metadata import SignalMetadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class MatchResult:
|
||||
"""Result of a signature match"""
|
||||
device_id: int
|
||||
device_name: str
|
||||
manufacturer: str
|
||||
confidence: float # 0.0 to 1.0
|
||||
match_method: str # 'exact', 'partial', 'pattern', 'timing'
|
||||
match_details: Dict[str, Any]
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'device_id': self.device_id,
|
||||
'device_name': self.device_name,
|
||||
'manufacturer': self.manufacturer,
|
||||
'confidence': self.confidence,
|
||||
'match_method': self.match_method,
|
||||
'match_details': self.match_details
|
||||
}
|
||||
|
||||
|
||||
class SignatureMatcher:
|
||||
"""
|
||||
Main signature matching engine
|
||||
|
||||
Coordinates multiple matching strategies to identify devices
|
||||
from RF signal metadata
|
||||
"""
|
||||
|
||||
def __init__(self, database):
|
||||
"""
|
||||
Initialize matcher with database connection
|
||||
|
||||
Args:
|
||||
database: Database connection for signature queries
|
||||
"""
|
||||
self.db = database
|
||||
self.strategies = []
|
||||
|
||||
def add_strategy(self, strategy):
|
||||
"""Add a matching strategy"""
|
||||
self.strategies.append(strategy)
|
||||
|
||||
def match(self, metadata: SignalMetadata, max_results: int = 10) -> List[MatchResult]:
|
||||
"""
|
||||
Match signal metadata against signature database
|
||||
|
||||
Args:
|
||||
metadata: Parsed signal metadata
|
||||
max_results: Maximum number of results to return
|
||||
|
||||
Returns:
|
||||
List of MatchResult objects sorted by confidence
|
||||
"""
|
||||
all_matches = []
|
||||
|
||||
# Run all matching strategies
|
||||
for strategy in self.strategies:
|
||||
try:
|
||||
matches = strategy.match(metadata, self.db)
|
||||
all_matches.extend(matches)
|
||||
except Exception as e:
|
||||
logger.error(f"Strategy {strategy.__class__.__name__} failed: {e}")
|
||||
|
||||
# Deduplicate and sort
|
||||
unique_matches = self._deduplicate_matches(all_matches)
|
||||
sorted_matches = sorted(unique_matches, key=lambda x: x.confidence, reverse=True)
|
||||
|
||||
return sorted_matches[:max_results]
|
||||
|
||||
def _deduplicate_matches(self, matches: List[MatchResult]) -> List[MatchResult]:
|
||||
"""
|
||||
Deduplicate matches, keeping highest confidence for each device
|
||||
|
||||
Args:
|
||||
matches: List of match results
|
||||
|
||||
Returns:
|
||||
Deduplicated list
|
||||
"""
|
||||
device_map = {}
|
||||
|
||||
for match in matches:
|
||||
if match.device_id not in device_map:
|
||||
device_map[match.device_id] = match
|
||||
else:
|
||||
# Keep higher confidence match
|
||||
if match.confidence > device_map[match.device_id].confidence:
|
||||
device_map[match.device_id] = match
|
||||
|
||||
return list(device_map.values())
|
||||
|
||||
|
||||
class MatchStrategy:
|
||||
"""Base class for matching strategies"""
|
||||
|
||||
def match(self, metadata: SignalMetadata, db) -> List[MatchResult]:
|
||||
"""
|
||||
Match metadata against database
|
||||
|
||||
Args:
|
||||
metadata: Signal metadata
|
||||
db: Database connection
|
||||
|
||||
Returns:
|
||||
List of MatchResult objects
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,358 @@
|
||||
"""
|
||||
Signature matching strategies
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
from loguru import logger
|
||||
|
||||
from .engine import MatchStrategy, MatchResult
|
||||
from ..parser.metadata import SignalMetadata
|
||||
|
||||
|
||||
class ExactMatcher(MatchStrategy):
|
||||
"""
|
||||
Exact matching strategy: protocol + frequency + bit length
|
||||
|
||||
Highest confidence (1.0) for perfect matches
|
||||
"""
|
||||
|
||||
def match(self, metadata: SignalMetadata, db) -> List[MatchResult]:
|
||||
"""Match by exact protocol, frequency, and bit length"""
|
||||
matches = []
|
||||
|
||||
if not metadata.protocol or not metadata.bit_length:
|
||||
return matches
|
||||
|
||||
# Query database for exact matches
|
||||
query = """
|
||||
SELECT DISTINCT
|
||||
d.id,
|
||||
d.manufacturer,
|
||||
d.model,
|
||||
s.protocol,
|
||||
s.frequency
|
||||
FROM devices d
|
||||
JOIN signatures s ON s.device_id = d.id
|
||||
WHERE s.protocol = %s
|
||||
AND s.frequency = %s
|
||||
AND (s.bit_length = %s OR s.bit_length IS NULL)
|
||||
"""
|
||||
|
||||
results = db.execute(query, (
|
||||
metadata.protocol,
|
||||
metadata.frequency,
|
||||
metadata.bit_length
|
||||
))
|
||||
|
||||
for row in results:
|
||||
matches.append(MatchResult(
|
||||
device_id=row['id'],
|
||||
device_name=row['model'],
|
||||
manufacturer=row['manufacturer'],
|
||||
confidence=1.0,
|
||||
match_method='exact',
|
||||
match_details={
|
||||
'protocol': row['protocol'],
|
||||
'frequency': row['frequency'],
|
||||
'bit_length': metadata.bit_length
|
||||
}
|
||||
))
|
||||
|
||||
logger.debug(f"ExactMatcher found {len(matches)} matches")
|
||||
return matches
|
||||
|
||||
|
||||
class PartialMatcher(MatchStrategy):
|
||||
"""
|
||||
Partial matching: protocol + frequency only
|
||||
|
||||
Confidence: 0.8
|
||||
"""
|
||||
|
||||
def match(self, metadata: SignalMetadata, db) -> List[MatchResult]:
|
||||
"""Match by protocol and frequency only"""
|
||||
matches = []
|
||||
|
||||
if not metadata.protocol:
|
||||
return matches
|
||||
|
||||
query = """
|
||||
SELECT DISTINCT
|
||||
d.id,
|
||||
d.manufacturer,
|
||||
d.model,
|
||||
s.protocol,
|
||||
s.frequency
|
||||
FROM devices d
|
||||
JOIN signatures s ON s.device_id = d.id
|
||||
WHERE s.protocol = %s
|
||||
AND s.frequency = %s
|
||||
"""
|
||||
|
||||
results = db.execute(query, (metadata.protocol, metadata.frequency))
|
||||
|
||||
for row in results:
|
||||
matches.append(MatchResult(
|
||||
device_id=row['id'],
|
||||
device_name=row['model'],
|
||||
manufacturer=row['manufacturer'],
|
||||
confidence=0.8,
|
||||
match_method='partial',
|
||||
match_details={
|
||||
'protocol': row['protocol'],
|
||||
'frequency': row['frequency']
|
||||
}
|
||||
))
|
||||
|
||||
logger.debug(f"PartialMatcher found {len(matches)} matches")
|
||||
return matches
|
||||
|
||||
|
||||
class PatternMatcher(MatchStrategy):
|
||||
"""
|
||||
Bit pattern matching with masks
|
||||
|
||||
Confidence: 0.7-0.9 based on pattern similarity
|
||||
"""
|
||||
|
||||
def match(self, metadata: SignalMetadata, db) -> List[MatchResult]:
|
||||
"""Match by bit pattern similarity"""
|
||||
matches = []
|
||||
|
||||
if not metadata.key_data:
|
||||
return matches
|
||||
|
||||
# Query signatures with bit patterns
|
||||
query = """
|
||||
SELECT DISTINCT
|
||||
d.id,
|
||||
d.manufacturer,
|
||||
d.model,
|
||||
s.bit_pattern,
|
||||
s.bit_mask,
|
||||
s.weight,
|
||||
s.protocol,
|
||||
s.frequency
|
||||
FROM devices d
|
||||
JOIN signatures s ON s.device_id = d.id
|
||||
WHERE s.bit_pattern IS NOT NULL
|
||||
AND s.frequency = %s
|
||||
"""
|
||||
|
||||
results = db.execute(query, (metadata.frequency,))
|
||||
|
||||
for row in results:
|
||||
# Apply mask and compare
|
||||
if row['bit_mask']:
|
||||
similarity = self._compare_with_mask(
|
||||
metadata.key_data,
|
||||
row['bit_pattern'],
|
||||
row['bit_mask']
|
||||
)
|
||||
else:
|
||||
similarity = self._compare_bytes(
|
||||
metadata.key_data,
|
||||
row['bit_pattern']
|
||||
)
|
||||
|
||||
if similarity > 0.5: # Minimum threshold
|
||||
confidence = similarity * row.get('weight', 1.0) * 0.9
|
||||
|
||||
matches.append(MatchResult(
|
||||
device_id=row['id'],
|
||||
device_name=row['model'],
|
||||
manufacturer=row['manufacturer'],
|
||||
confidence=confidence,
|
||||
match_method='pattern',
|
||||
match_details={
|
||||
'protocol': row['protocol'],
|
||||
'frequency': row['frequency'],
|
||||
'similarity': similarity
|
||||
}
|
||||
))
|
||||
|
||||
logger.debug(f"PatternMatcher found {len(matches)} matches")
|
||||
return matches
|
||||
|
||||
def _compare_with_mask(self, data1: bytes, data2: bytes, mask: bytes) -> float:
|
||||
"""
|
||||
Compare two byte arrays with a mask
|
||||
|
||||
Args:
|
||||
data1: First byte array
|
||||
data2: Second byte array
|
||||
mask: Mask (1=compare, 0=ignore)
|
||||
|
||||
Returns:
|
||||
Similarity score (0.0 to 1.0)
|
||||
"""
|
||||
if not data1 or not data2 or not mask:
|
||||
return 0.0
|
||||
|
||||
# Ensure same length
|
||||
min_len = min(len(data1), len(data2), len(mask))
|
||||
|
||||
matching_bits = 0
|
||||
total_bits = 0
|
||||
|
||||
for i in range(min_len):
|
||||
mask_byte = mask[i] if i < len(mask) else 0xFF
|
||||
|
||||
# Count bits to compare (1s in mask)
|
||||
bits_to_check = bin(mask_byte).count('1')
|
||||
total_bits += bits_to_check
|
||||
|
||||
# Compare masked bytes
|
||||
masked1 = data1[i] & mask_byte
|
||||
masked2 = data2[i] & mask_byte
|
||||
|
||||
# Count matching bits
|
||||
xor_result = masked1 ^ masked2
|
||||
matching_bits += bits_to_check - bin(xor_result).count('1')
|
||||
|
||||
if total_bits == 0:
|
||||
return 0.0
|
||||
|
||||
return matching_bits / total_bits
|
||||
|
||||
def _compare_bytes(self, data1: bytes, data2: bytes) -> float:
|
||||
"""
|
||||
Compare two byte arrays directly
|
||||
|
||||
Returns:
|
||||
Similarity score (0.0 to 1.0)
|
||||
"""
|
||||
if not data1 or not data2:
|
||||
return 0.0
|
||||
|
||||
min_len = min(len(data1), len(data2))
|
||||
matches = sum(1 for i in range(min_len) if data1[i] == data2[i])
|
||||
|
||||
return matches / max(len(data1), len(data2))
|
||||
|
||||
|
||||
class TimingMatcher(MatchStrategy):
|
||||
"""
|
||||
Timing pattern matching for RAW signals
|
||||
|
||||
Confidence: 0.6-0.8 based on timing similarity
|
||||
"""
|
||||
|
||||
def match(self, metadata: SignalMetadata, db) -> List[MatchResult]:
|
||||
"""Match by timing pattern characteristics"""
|
||||
matches = []
|
||||
|
||||
if not metadata.raw_data or not metadata.avg_pulse_width:
|
||||
return matches
|
||||
|
||||
# Query signatures with timing information
|
||||
query = """
|
||||
SELECT DISTINCT
|
||||
d.id,
|
||||
d.manufacturer,
|
||||
d.model,
|
||||
s.timing_min,
|
||||
s.timing_max,
|
||||
s.protocol,
|
||||
s.frequency
|
||||
FROM devices d
|
||||
JOIN signatures s ON s.device_id = d.id
|
||||
WHERE s.timing_min IS NOT NULL
|
||||
AND s.frequency = %s
|
||||
"""
|
||||
|
||||
results = db.execute(query, (metadata.frequency,))
|
||||
|
||||
avg_pulse = metadata.avg_pulse_width
|
||||
|
||||
for row in results:
|
||||
timing_min = row['timing_min']
|
||||
timing_max = row['timing_max']
|
||||
|
||||
# Check if average pulse width falls within range
|
||||
if timing_min <= avg_pulse <= timing_max:
|
||||
# Calculate confidence based on how centered the value is
|
||||
range_size = timing_max - timing_min
|
||||
center = (timing_max + timing_min) / 2
|
||||
distance_from_center = abs(avg_pulse - center)
|
||||
|
||||
# Confidence decreases as we move away from center
|
||||
confidence = 0.8 * (1.0 - (distance_from_center / (range_size / 2)))
|
||||
confidence = max(0.6, min(0.8, confidence))
|
||||
|
||||
matches.append(MatchResult(
|
||||
device_id=row['id'],
|
||||
device_name=row['model'],
|
||||
manufacturer=row['manufacturer'],
|
||||
confidence=confidence,
|
||||
match_method='timing',
|
||||
match_details={
|
||||
'protocol': row['protocol'],
|
||||
'frequency': row['frequency'],
|
||||
'avg_pulse_width': avg_pulse,
|
||||
'timing_range': (timing_min, timing_max)
|
||||
}
|
||||
))
|
||||
|
||||
logger.debug(f"TimingMatcher found {len(matches)} matches")
|
||||
return matches
|
||||
|
||||
|
||||
class FrequencyMatcher(MatchStrategy):
|
||||
"""
|
||||
Fuzzy frequency matching (within tolerance)
|
||||
|
||||
Confidence: 0.5-0.7 based on frequency proximity
|
||||
"""
|
||||
|
||||
def __init__(self, tolerance_hz: int = 5000):
|
||||
"""
|
||||
Initialize frequency matcher
|
||||
|
||||
Args:
|
||||
tolerance_hz: Frequency tolerance in Hz (default 5kHz)
|
||||
"""
|
||||
self.tolerance = tolerance_hz
|
||||
|
||||
def match(self, metadata: SignalMetadata, db) -> List[MatchResult]:
|
||||
"""Match by frequency proximity"""
|
||||
matches = []
|
||||
|
||||
query = """
|
||||
SELECT DISTINCT
|
||||
d.id,
|
||||
d.manufacturer,
|
||||
d.model,
|
||||
s.frequency,
|
||||
s.protocol
|
||||
FROM devices d
|
||||
JOIN signatures s ON s.device_id = d.id
|
||||
WHERE s.frequency BETWEEN %s AND %s
|
||||
AND (s.protocol IS NULL OR s.protocol = 'Unknown')
|
||||
"""
|
||||
|
||||
freq_min = metadata.frequency - self.tolerance
|
||||
freq_max = metadata.frequency + self.tolerance
|
||||
|
||||
results = db.execute(query, (freq_min, freq_max))
|
||||
|
||||
for row in results:
|
||||
# Calculate confidence based on frequency difference
|
||||
freq_diff = abs(row['frequency'] - metadata.frequency)
|
||||
confidence = 0.7 * (1.0 - (freq_diff / self.tolerance))
|
||||
confidence = max(0.5, min(0.7, confidence))
|
||||
|
||||
matches.append(MatchResult(
|
||||
device_id=row['id'],
|
||||
device_name=row['model'],
|
||||
manufacturer=row['manufacturer'],
|
||||
confidence=confidence,
|
||||
match_method='frequency',
|
||||
match_details={
|
||||
'frequency': row['frequency'],
|
||||
'frequency_diff_hz': freq_diff
|
||||
}
|
||||
))
|
||||
|
||||
logger.debug(f"FrequencyMatcher found {len(matches)} matches")
|
||||
return matches
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
RF signal file parsers for .sub, .fff, and other formats
|
||||
"""
|
||||
|
||||
from .sub_parser import SubFileParser, parse_sub_file
|
||||
from .metadata import SignalMetadata
|
||||
|
||||
__all__ = ['SubFileParser', 'parse_sub_file', 'SignalMetadata']
|
||||
@@ -0,0 +1,102 @@
|
||||
"""
|
||||
Signal metadata data structures
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class SignalMetadata:
|
||||
"""
|
||||
Extracted metadata from RF signal capture file
|
||||
"""
|
||||
# File information
|
||||
file_type: str # 'Flipper SubGhz Key File' or 'Flipper SubGhz RAW File'
|
||||
version: int
|
||||
file_format: str # 'KEY', 'RAW', 'BinRAW'
|
||||
|
||||
# RF characteristics
|
||||
frequency: int # in Hz
|
||||
preset: Optional[str] = None
|
||||
modulation: Optional[str] = None # Extracted from preset
|
||||
|
||||
# Protocol information (for decoded files)
|
||||
protocol: Optional[str] = None
|
||||
bit_length: Optional[int] = None
|
||||
key_data: Optional[bytes] = None
|
||||
timing_element: Optional[int] = None # TE in microseconds
|
||||
|
||||
# RAW signal data
|
||||
raw_data: Optional[List[int]] = None # Timing array
|
||||
|
||||
# BinRAW data
|
||||
bin_raw_bit: Optional[int] = None
|
||||
bin_raw_te: Optional[int] = None
|
||||
bin_raw_data: Optional[bytes] = None
|
||||
|
||||
# Custom preset data
|
||||
custom_preset_module: Optional[str] = None
|
||||
custom_preset_data: Optional[bytes] = None
|
||||
|
||||
# Computed fields
|
||||
duration_ms: Optional[float] = None
|
||||
pulse_count: Optional[int] = None
|
||||
avg_pulse_width: Optional[float] = None
|
||||
|
||||
# Metadata
|
||||
parsed_at: datetime = field(default_factory=datetime.utcnow)
|
||||
parse_errors: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
"""Convert to dictionary for JSON serialization"""
|
||||
return {
|
||||
'file_type': self.file_type,
|
||||
'version': self.version,
|
||||
'file_format': self.file_format,
|
||||
'frequency': self.frequency,
|
||||
'preset': self.preset,
|
||||
'modulation': self.modulation,
|
||||
'protocol': self.protocol,
|
||||
'bit_length': self.bit_length,
|
||||
'key_data': self.key_data.hex() if self.key_data else None,
|
||||
'timing_element': self.timing_element,
|
||||
'raw_data_length': len(self.raw_data) if self.raw_data else 0,
|
||||
'duration_ms': self.duration_ms,
|
||||
'pulse_count': self.pulse_count,
|
||||
'avg_pulse_width': self.avg_pulse_width,
|
||||
'parsed_at': self.parsed_at.isoformat(),
|
||||
'parse_errors': self.parse_errors
|
||||
}
|
||||
|
||||
@property
|
||||
def is_decoded(self) -> bool:
|
||||
"""Check if signal has been decoded to a protocol"""
|
||||
return self.protocol is not None and self.protocol != 'RAW'
|
||||
|
||||
@property
|
||||
def has_raw_data(self) -> bool:
|
||||
"""Check if file contains raw timing data"""
|
||||
return self.raw_data is not None and len(self.raw_data) > 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class PresetInfo:
|
||||
"""Modulation preset information"""
|
||||
name: str
|
||||
modulation: str # 'OOK' or '2FSK'
|
||||
bandwidth_khz: float
|
||||
deviation_khz: Optional[float] = None # For FSK
|
||||
|
||||
@staticmethod
|
||||
def from_preset_name(name: str) -> 'PresetInfo':
|
||||
"""Parse preset information from name"""
|
||||
presets = {
|
||||
'FuriHalSubGhzPresetOok270Async': PresetInfo('OOK270', 'OOK', 270.0),
|
||||
'FuriHalSubGhzPresetOok650Async': PresetInfo('OOK650', 'OOK', 650.0),
|
||||
'FuriHalSubGhzPreset2FSKDev238Async': PresetInfo('FM238', '2FSK', 270.0, 2.38),
|
||||
'FuriHalSubGhzPreset2FSKDev476Async': PresetInfo('FM476', '2FSK', 270.0, 47.6),
|
||||
'FuriHalSubGhzPresetCustom': PresetInfo('Custom', 'Custom', 0.0),
|
||||
}
|
||||
return presets.get(name, PresetInfo('Unknown', 'Unknown', 0.0))
|
||||
@@ -0,0 +1,307 @@
|
||||
"""
|
||||
Flipper Zero .sub file parser
|
||||
|
||||
Parses both KEY and RAW format .sub files and extracts signal metadata
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, List
|
||||
from loguru import logger
|
||||
|
||||
from .metadata import SignalMetadata, PresetInfo
|
||||
|
||||
|
||||
class SubFileParser:
|
||||
"""Parser for Flipper Zero .sub files"""
|
||||
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
"""Reset parser state"""
|
||||
self.fields = {}
|
||||
self.errors = []
|
||||
|
||||
def parse(self, file_path: str) -> SignalMetadata:
|
||||
"""
|
||||
Parse a .sub file and extract metadata
|
||||
|
||||
Args:
|
||||
file_path: Path to .sub file
|
||||
|
||||
Returns:
|
||||
SignalMetadata object
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
ValueError: If file format is invalid
|
||||
"""
|
||||
self.reset()
|
||||
path = Path(file_path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Read file
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to read file: {e}")
|
||||
|
||||
# Parse fields
|
||||
for line in content.split('\n'):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
|
||||
if ':' in line:
|
||||
key, value = line.split(':', 1)
|
||||
self.fields[key.strip()] = value.strip()
|
||||
|
||||
# Validate required fields
|
||||
if 'Filetype' not in self.fields:
|
||||
raise ValueError("Missing required field: Filetype")
|
||||
|
||||
if 'Frequency' not in self.fields:
|
||||
raise ValueError("Missing required field: Frequency")
|
||||
|
||||
# Extract metadata
|
||||
return self._extract_metadata()
|
||||
|
||||
def _extract_metadata(self) -> SignalMetadata:
|
||||
"""Extract metadata from parsed fields"""
|
||||
|
||||
# Basic fields
|
||||
file_type = self.fields.get('Filetype', '')
|
||||
version = int(self.fields.get('Version', 1))
|
||||
frequency = int(self.fields.get('Frequency', 0))
|
||||
preset = self.fields.get('Preset')
|
||||
|
||||
# Determine file format
|
||||
protocol = self.fields.get('Protocol', 'Unknown')
|
||||
if protocol == 'RAW':
|
||||
file_format = 'RAW'
|
||||
elif protocol == 'BinRAW':
|
||||
file_format = 'BinRAW'
|
||||
else:
|
||||
file_format = 'KEY'
|
||||
|
||||
# Extract modulation from preset
|
||||
modulation = None
|
||||
if preset:
|
||||
preset_info = PresetInfo.from_preset_name(preset)
|
||||
modulation = preset_info.modulation
|
||||
|
||||
# Initialize metadata
|
||||
metadata = SignalMetadata(
|
||||
file_type=file_type,
|
||||
version=version,
|
||||
file_format=file_format,
|
||||
frequency=frequency,
|
||||
preset=preset,
|
||||
modulation=modulation,
|
||||
protocol=protocol if protocol not in ['RAW', 'BinRAW'] else None
|
||||
)
|
||||
|
||||
# Extract format-specific fields
|
||||
if file_format == 'KEY':
|
||||
self._extract_key_fields(metadata)
|
||||
elif file_format == 'RAW':
|
||||
self._extract_raw_fields(metadata)
|
||||
elif file_format == 'BinRAW':
|
||||
self._extract_binraw_fields(metadata)
|
||||
|
||||
# Extract custom preset if present
|
||||
if 'Custom_preset_module' in self.fields:
|
||||
metadata.custom_preset_module = self.fields['Custom_preset_module']
|
||||
if 'Custom_preset_data' in self.fields:
|
||||
metadata.custom_preset_data = bytes.fromhex(
|
||||
self.fields['Custom_preset_data'].replace(' ', '')
|
||||
)
|
||||
|
||||
# Add any parsing errors
|
||||
metadata.parse_errors = self.errors
|
||||
|
||||
return metadata
|
||||
|
||||
def _extract_key_fields(self, metadata: SignalMetadata):
|
||||
"""Extract fields from KEY format file"""
|
||||
try:
|
||||
if 'Bit' in self.fields:
|
||||
metadata.bit_length = int(self.fields['Bit'])
|
||||
|
||||
if 'Key' in self.fields:
|
||||
# Parse hex key data
|
||||
key_hex = self.fields['Key'].replace(' ', '')
|
||||
metadata.key_data = bytes.fromhex(key_hex)
|
||||
|
||||
if 'TE' in self.fields:
|
||||
metadata.timing_element = int(self.fields['TE'])
|
||||
|
||||
except ValueError as e:
|
||||
self.errors.append(f"Error parsing KEY fields: {e}")
|
||||
logger.warning(f"Parse error: {e}")
|
||||
|
||||
def _extract_raw_fields(self, metadata: SignalMetadata):
|
||||
"""Extract fields from RAW format file"""
|
||||
try:
|
||||
if 'RAW_Data' in self.fields:
|
||||
# Parse timing array
|
||||
raw_str = self.fields['RAW_Data']
|
||||
timings = [int(x) for x in raw_str.split()]
|
||||
metadata.raw_data = timings
|
||||
|
||||
# Calculate statistics
|
||||
metadata.pulse_count = len(timings)
|
||||
metadata.duration_ms = sum(abs(t) for t in timings) / 1000.0
|
||||
|
||||
# Average pulse width (positive values only)
|
||||
positive_pulses = [t for t in timings if t > 0]
|
||||
if positive_pulses:
|
||||
metadata.avg_pulse_width = sum(positive_pulses) / len(positive_pulses)
|
||||
|
||||
except ValueError as e:
|
||||
self.errors.append(f"Error parsing RAW fields: {e}")
|
||||
logger.warning(f"Parse error: {e}")
|
||||
|
||||
def _extract_binraw_fields(self, metadata: SignalMetadata):
|
||||
"""Extract fields from BinRAW format file"""
|
||||
try:
|
||||
if 'Bit' in self.fields:
|
||||
metadata.bit_length = int(self.fields['Bit'])
|
||||
|
||||
if 'TE' in self.fields:
|
||||
metadata.bin_raw_te = int(self.fields['TE'])
|
||||
|
||||
if 'Bit_RAW' in self.fields:
|
||||
metadata.bin_raw_bit = int(self.fields['Bit_RAW'])
|
||||
|
||||
if 'Data_RAW' in self.fields:
|
||||
data_hex = self.fields['Data_RAW'].replace(' ', '')
|
||||
metadata.bin_raw_data = bytes.fromhex(data_hex)
|
||||
|
||||
# Calculate duration
|
||||
if metadata.bit_length and metadata.bin_raw_te:
|
||||
metadata.duration_ms = (metadata.bit_length * metadata.bin_raw_te) / 1000.0
|
||||
|
||||
except ValueError as e:
|
||||
self.errors.append(f"Error parsing BinRAW fields: {e}")
|
||||
logger.warning(f"Parse error: {e}")
|
||||
|
||||
|
||||
def parse_sub_file(file_path: str) -> SignalMetadata:
|
||||
"""
|
||||
Convenience function to parse a .sub file
|
||||
|
||||
Args:
|
||||
file_path: Path to .sub file
|
||||
|
||||
Returns:
|
||||
SignalMetadata object
|
||||
"""
|
||||
parser = SubFileParser()
|
||||
return parser.parse(file_path)
|
||||
|
||||
|
||||
def extract_protocol_features(metadata: SignalMetadata) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract features for protocol matching
|
||||
|
||||
Args:
|
||||
metadata: Parsed signal metadata
|
||||
|
||||
Returns:
|
||||
Dictionary of matchable features
|
||||
"""
|
||||
features = {
|
||||
'frequency': metadata.frequency,
|
||||
'modulation': metadata.modulation,
|
||||
'protocol': metadata.protocol,
|
||||
}
|
||||
|
||||
if metadata.is_decoded:
|
||||
# For decoded signals
|
||||
features.update({
|
||||
'bit_length': metadata.bit_length,
|
||||
'timing_element': metadata.timing_element,
|
||||
'key_pattern': metadata.key_data[:4] if metadata.key_data else None # First 4 bytes
|
||||
})
|
||||
|
||||
elif metadata.has_raw_data:
|
||||
# For RAW signals, extract timing characteristics
|
||||
features.update({
|
||||
'pulse_count': metadata.pulse_count,
|
||||
'avg_pulse_width': metadata.avg_pulse_width,
|
||||
'duration_ms': metadata.duration_ms,
|
||||
'timing_pattern': _extract_timing_pattern(metadata.raw_data)
|
||||
})
|
||||
|
||||
return features
|
||||
|
||||
|
||||
def _extract_timing_pattern(raw_data: List[int], max_samples: int = 20) -> List[int]:
|
||||
"""
|
||||
Extract representative timing pattern from RAW data
|
||||
|
||||
Args:
|
||||
raw_data: Array of timing values
|
||||
max_samples: Maximum number of samples to return
|
||||
|
||||
Returns:
|
||||
Representative timing pattern
|
||||
"""
|
||||
if not raw_data:
|
||||
return []
|
||||
|
||||
# Take first few pulses as pattern
|
||||
pattern = raw_data[:max_samples]
|
||||
|
||||
# Normalize to absolute values
|
||||
return [abs(t) for t in pattern]
|
||||
|
||||
|
||||
def validate_sub_file(file_path: str) -> tuple[bool, List[str]]:
|
||||
"""
|
||||
Validate a .sub file without full parsing
|
||||
|
||||
Args:
|
||||
file_path: Path to .sub file
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_messages)
|
||||
"""
|
||||
errors = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Check for required header
|
||||
if 'Filetype:' not in content:
|
||||
errors.append("Missing Filetype field")
|
||||
|
||||
if 'Frequency:' not in content:
|
||||
errors.append("Missing Frequency field")
|
||||
|
||||
if 'Version:' not in content:
|
||||
errors.append("Missing Version field")
|
||||
|
||||
# Check file type
|
||||
if 'Flipper SubGhz' not in content:
|
||||
errors.append("Not a valid Flipper SubGhz file")
|
||||
|
||||
# Validate frequency value
|
||||
freq_match = re.search(r'Frequency:\s*(\d+)', content)
|
||||
if freq_match:
|
||||
freq = int(freq_match.group(1))
|
||||
if freq < 300_000_000 or freq > 928_000_000:
|
||||
errors.append(f"Frequency out of range: {freq} Hz")
|
||||
|
||||
except FileNotFoundError:
|
||||
errors.append("File not found")
|
||||
except Exception as e:
|
||||
errors.append(f"Validation error: {e}")
|
||||
|
||||
return (len(errors) == 0, errors)
|
||||
Reference in New Issue
Block a user