Trilltechnician/giglez
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: Phase 1 & Phase 2 infrastructure complete

Browse Source
This commit is contained in:
priestlypython
2026-01-12 11:21:17 -08:00
commit eb225771bc
53 changed files with 13645 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/__init__.py
+9
View File
@@ -0,0 +1,9 @@
"""
GigLez API Package
FastAPI application with environment-aware configuration
"""
from .main import app
__all__ = ['app']
src/api/dependencies.py
+185
View File
@@ -0,0 +1,185 @@
"""
FastAPI Dependencies
Shared dependencies for API endpoints
"""
from typing import Generator, Optional
from fastapi import Depends, HTTPException, status, Security
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from sqlalchemy.orm import Session
from config.database import get_db_session
from config.settings import settings
from src.database.models import User
# =============================================================================
# DATABASE DEPENDENCY
# =============================================================================
def get_db() -> Generator[Session, None, None]:
"""
Database session dependency
Yields database session and ensures it's closed after request
"""
session = get_db_session()
try:
yield session
finally:
session.close()
# =============================================================================
# AUTHENTICATION DEPENDENCIES
# =============================================================================
# HTTP Bearer token scheme
security = HTTPBearer(auto_error=False)
def get_current_user(
credentials: Optional[HTTPAuthorizationCredentials] = Security(security),
db: Session = Depends(get_db)
) -> Optional[User]:
"""
Get current authenticated user from JWT token
Args:
credentials: Bearer token from Authorization header
db: Database session
Returns:
User object if authenticated, None if auth not required
Raises:
HTTPException: If authentication required but invalid
"""
# If authentication not required (development mode)
if not settings.require_auth:
# Return anonymous user (ID 0)
return db.query(User).filter_by(id=0).first()
# Authentication required
if not credentials:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Authentication required",
headers={"WWW-Authenticate": "Bearer"},
)
try:
# Decode JWT token (implementation in auth.py)
from src.api.auth import decode_access_token
payload = decode_access_token(credentials.credentials)
user_id = payload.get("sub")
if user_id is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authentication token"
)
# Get user from database
user = db.query(User).filter_by(id=int(user_id)).first()
if user is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found"
)
return user
except Exception as e:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=f"Authentication failed: {str(e)}"
)
def require_auth(user: User = Depends(get_current_user)) -> User:
"""
Require authentication (always)
Use this dependency when endpoint must have authentication
regardless of global settings
Args:
user: Current user from get_current_user
Returns:
Authenticated user
Raises:
HTTPException: If user not authenticated
"""
if user is None or user.id == 0:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Authentication required for this endpoint"
)
return user
def optional_auth(user: Optional[User] = Depends(get_current_user)) -> Optional[User]:
"""
Optional authentication
Returns user if authenticated, None otherwise (no error)
Args:
user: Current user from get_current_user
Returns:
User if authenticated, None otherwise
"""
return user
# =============================================================================
# PAGINATION DEPENDENCIES
# =============================================================================
def get_pagination(
skip: int = 0,
limit: int = 100
) -> dict:
"""
Pagination parameters
Args:
skip: Number of records to skip
limit: Maximum number of records to return
Returns:
Dictionary with skip and limit
"""
# Enforce maximum limit
max_limit = 1000
if limit > max_limit:
limit = max_limit
return {
"skip": skip,
"limit": limit
}
# =============================================================================
# STORAGE DEPENDENCY
# =============================================================================
def get_storage():
"""
Get storage backend
Returns:
Storage backend instance (LocalStorage or S3Storage)
"""
from src.core.storage import get_storage
return get_storage()
src/api/main.py
+262
View File
@@ -0,0 +1,262 @@
"""
GigLez FastAPI Application
Environment-aware API server supporting both development (Termux) and production (Server) modes
"""
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse
from contextlib import asynccontextmanager
from loguru import logger
import time
from config.settings import settings
from config.database import get_db_config
# =============================================================================
# LIFESPAN MANAGEMENT
# =============================================================================
@asynccontextmanager
async def lifespan(app: FastAPI):
"""
Application lifespan manager
Handles startup and shutdown events
"""
# Startup
logger.info("=" * 80)
logger.info("GigLez API Starting")
logger.info("=" * 80)
# Display configuration
settings.display()
# Validate configuration
errors = settings.validate()
if errors:
logger.error("Configuration validation failed:")
for error in errors:
logger.error(f" ❌ {error}")
raise RuntimeError("Invalid configuration")
# Test database connection
db_config = get_db_config()
if not db_config.test_connection():
raise RuntimeError("Database connection failed")
if not db_config.test_postgis():
raise RuntimeError("PostGIS extension not available")
logger.success("✅ Database connection established")
logger.success("✅ PostGIS extension available")
# Initialize storage backend
from src.core.storage import get_storage_backend
storage = get_storage_backend()
logger.success(f"✅ Storage backend initialized: {type(storage).__name__}")
# Initialize background task system
if settings.use_celery:
logger.info("Using Celery for background tasks")
# Celery app will be imported when needed
else:
logger.info("Using synchronous background tasks")
logger.success("=" * 80)
logger.success(f"🚀 GigLez API ready on {settings.api_host}:{settings.api_port}")
logger.success(f"📡 Mode: {settings.mode.value}")
logger.success("=" * 80)
yield
# Shutdown
logger.info("=" * 80)
logger.info("GigLez API Shutting Down")
logger.info("=" * 80)
# Close database connections
db_config.close()
logger.success("✅ Database connections closed")
logger.info("Shutdown complete")
# =============================================================================
# APPLICATION INSTANCE
# =============================================================================
app = FastAPI(
title="GigLez API",
description="IoT RF Device Mapping Platform - Wigle for Sub-GHz Signals",
version="1.0.0",
lifespan=lifespan,
docs_url="/docs" if settings.debug_endpoints else None,
redoc_url="/redoc" if settings.debug_endpoints else None,
openapi_url="/openapi.json" if settings.debug_endpoints else None,
)
# =============================================================================
# MIDDLEWARE
# =============================================================================
# CORS Middleware
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
expose_headers=["X-Total-Count", "X-Page-Size"],
)
# GZip compression
app.add_middleware(GZipMiddleware, minimum_size=1000)
# Request timing middleware
@app.middleware("http")
async def add_process_time_header(request: Request, call_next):
"""Add request processing time header"""
start_time = time.time()
response = await call_next(request)
process_time = time.time() - start_time
response.headers["X-Process-Time"] = f"{process_time:.4f}"
return response
# Request logging middleware
@app.middleware("http")
async def log_requests(request: Request, call_next):
"""Log all requests"""
logger.info(f"{request.method} {request.url.path}")
response = await call_next(request)
logger.info(f"{request.method} {request.url.path} - {response.status_code}")
return response
# =============================================================================
# EXCEPTION HANDLERS
# =============================================================================
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
"""Global exception handler"""
logger.error(f"Unhandled exception: {exc}", exc_info=True)
if settings.debug_errors:
# Detailed error in development
return JSONResponse(
status_code=500,
content={
"error": "Internal Server Error",
"detail": str(exc),
"type": type(exc).__name__
}
)
else:
# Generic error in production
return JSONResponse(
status_code=500,
content={"error": "Internal Server Error"}
)
# =============================================================================
# ROOT ENDPOINTS
# =============================================================================
@app.get("/")
async def root():
"""Root endpoint - API information"""
return {
"name": "GigLez API",
"version": "1.0.0",
"description": "IoT RF Device Mapping Platform",
"mode": settings.mode.value,
"docs": "/docs" if settings.debug_endpoints else "disabled",
"status": "operational"
}
@app.get("/health")
async def health():
"""Health check endpoint"""
# Test database connection
db_config = get_db_config()
db_healthy = db_config.test_connection()
return {
"status": "healthy" if db_healthy else "unhealthy",
"database": "connected" if db_healthy else "disconnected",
"mode": settings.mode.value,
"hardware_enabled": settings.enable_hardware
}
@app.get("/version")
async def version():
"""Version information"""
return {
"version": "1.0.0",
"mode": settings.mode.value,
"api": "v1"
}
# =============================================================================
# IMPORT ROUTERS
# =============================================================================
# Import and register route modules
from src.api.routes import captures, devices, query, stats
# Register routers
app.include_router(captures.router, prefix="/api/v1", tags=["captures"])
app.include_router(devices.router, prefix="/api/v1", tags=["devices"])
app.include_router(query.router, prefix="/api/v1", tags=["query"])
app.include_router(stats.router, prefix="/api/v1", tags=["statistics"])
# Hardware endpoints (development mode only)
if settings.enable_hardware:
from src.api.routes import hardware
app.include_router(hardware.router, prefix="/api/v1", tags=["hardware"])
logger.info("✅ Hardware endpoints enabled")
# Debug endpoints (development mode only)
if settings.debug_endpoints:
from src.api.routes import debug
app.include_router(debug.router, prefix="/api/debug", tags=["debug"])
logger.info("✅ Debug endpoints enabled")
# =============================================================================
# STARTUP MESSAGE
# =============================================================================
logger.info("FastAPI application initialized")
logger.info(f"Mode: {settings.mode.value}")
logger.info(f"Authentication: {'Required' if settings.require_auth else 'Optional'}")
logger.info(f"CORS Origins: {settings.cors_origins}")
logger.info(f"Storage: {settings.storage_type.value}")
# =============================================================================
# RUN WITH UVICORN (for development)
# =============================================================================
if __name__ == "__main__":
import uvicorn
uvicorn.run(
"src.api.main:app",
host=settings.api_host,
port=settings.api_port,
reload=settings.auto_reload,
log_level=settings.log_level.lower(),
access_log=True,
)
src/api/routes/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""
GigLez API Routes
Route modules for different API endpoints
"""
src/api/routes/captures.py
+267
View File
@@ -0,0 +1,267 @@
"""
Capture Upload Endpoints
Handles .sub file uploads with GPS manifests
"""
from typing import List
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status
from sqlalchemy.orm import Session
from loguru import logger
import hashlib
import json
from datetime import datetime
from src.api.dependencies import get_db, optional_auth, get_storage
from src.database.models import User, Capture, Session as CaptureSession
from src.parser.sub_parser import parse_sub_file
from src.gps.validator import validate_gps_coordinates
from config.settings import settings
router = APIRouter()
# =============================================================================
# UPLOAD ENDPOINT
# =============================================================================
@router.post("/captures/upload")
async def upload_captures(
manifest: str = Form(..., description="JSON manifest with GPS data"),
files: List[UploadFile] = File(..., description=".sub files to upload"),
db: Session = Depends(get_db),
user: User = Depends(optional_auth),
storage = Depends(get_storage)
):
"""
Upload RF captures with GPS coordinates
**Manifest Format** (JSON):
```json
{
"session_uuid": "550e8400-e29b-41d4-a716-446655440000",
"captures": [
{
"filename": "capture_001.sub",
"latitude": 40.7128,
"longitude": -74.0060,
"accuracy": 5.0,
"altitude": 10.5,
"timestamp": "2026-01-12T10:00:00Z"
}
]
}
```
**Process**:
1. Parse and validate manifest
2. Compute SHA256 hash for each file (deduplication)
3. Validate GPS coordinates
4. Parse .sub file metadata
5. Store file in storage backend
6. Save capture to database
7. Return upload summary
Returns:
Upload summary with processed captures
"""
try:
# Parse manifest
manifest_data = json.loads(manifest)
# Extract session UUID
session_uuid = manifest_data.get("session_uuid")
captures_manifest = manifest_data.get("captures", [])
logger.info(f"Upload request: {len(files)} files, session={session_uuid}")
# Get or create session
capture_session = None
if session_uuid:
capture_session = db.query(CaptureSession).filter_by(
session_uuid=session_uuid
).first()
if not capture_session:
# Create new session
capture_session = CaptureSession(
session_uuid=session_uuid,
user_id=user.id if user else None,
started_at=datetime.utcnow()
)
db.add(capture_session)
db.flush() # Get session ID
# Process files
results = []
errors = []
# Create filename to manifest mapping
manifest_map = {c['filename']: c for c in captures_manifest}
for file in files:
try:
# Read file content
content = await file.read()
# Compute SHA256 hash
file_hash = hashlib.sha256(content).hexdigest()
# Check if already exists (deduplication)
existing = db.query(Capture).filter_by(file_hash=file_hash).first()
if existing:
results.append({
"filename": file.filename,
"status": "duplicate",
"file_hash": file_hash
})
continue
# Get manifest entry for this file
manifest_entry = manifest_map.get(file.filename)
if not manifest_entry:
errors.append({
"filename": file.filename,
"error": "Missing manifest entry"
})
continue
# Validate GPS coordinates
latitude = float(manifest_entry['latitude'])
longitude = float(manifest_entry['longitude'])
accuracy = manifest_entry.get('accuracy')
if not validate_gps_coordinates(latitude, longitude, accuracy):
errors.append({
"filename": file.filename,
"error": f"Invalid GPS coordinates: ({latitude}, {longitude})"
})
continue
# Parse .sub file
# Save content temporarily to parse
import tempfile
with tempfile.NamedTemporaryFile(delete=False, suffix='.sub') as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
metadata = parse_sub_file(tmp_path)
except Exception as e:
errors.append({
"filename": file.filename,
"error": f"Failed to parse .sub file: {e}"
})
continue
finally:
import os
os.unlink(tmp_path)
# Store file in storage backend
storage_path = storage.save_file(
file_hash=file_hash,
content=content,
filename=file.filename
)
# Create capture record
capture = Capture(
file_hash=file_hash,
session_id=capture_session.id if capture_session else None,
user_id=user.id if user else None,
latitude=latitude,
longitude=longitude,
altitude=manifest_entry.get('altitude'),
gps_accuracy=accuracy,
captured_at=datetime.fromisoformat(
manifest_entry['timestamp'].replace('Z', '+00:00')
),
frequency=metadata.frequency,
rssi=None, # Not in .sub file typically
modulation=metadata.modulation,
preset=metadata.preset,
protocol=metadata.protocol,
bit_length=metadata.bit_length,
key_data=metadata.key_data,
timing_element=metadata.timing_element,
raw_data=str(metadata.raw_data) if metadata.raw_data else None,
raw_format=metadata.file_format,
file_path=storage_path,
file_size=len(content)
)
db.add(capture)
results.append({
"filename": file.filename,
"status": "uploaded",
"file_hash": file_hash,
"frequency": metadata.frequency,
"protocol": metadata.protocol
})
logger.debug(f"Capture uploaded: {file.filename} -> {file_hash}")
except Exception as e:
logger.error(f"Failed to process file {file.filename}: {e}")
errors.append({
"filename": file.filename,
"error": str(e)
})
# Commit all captures
db.commit()
logger.success(f"Upload complete: {len(results)} processed, {len(errors)} errors")
return {
"session_uuid": session_uuid,
"uploaded": len([r for r in results if r['status'] == 'uploaded']),
"duplicates": len([r for r in results if r['status'] == 'duplicate']),
"errors": len(errors),
"results": results,
"error_details": errors if errors else None
}
except json.JSONDecodeError as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid JSON manifest: {e}"
)
except Exception as e:
logger.error(f"Upload failed: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Upload failed: {e}"
)
# =============================================================================
# CAPTURE QUERY ENDPOINT
# =============================================================================
@router.get("/captures/{file_hash}")
async def get_capture(
file_hash: str,
db: Session = Depends(get_db)
):
"""
Get capture details by file hash
Args:
file_hash: SHA256 hash of capture file
Returns:
Capture details including metadata and device match
"""
capture = db.query(Capture).filter_by(file_hash=file_hash).first()
if not capture:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Capture not found: {file_hash}"
)
return capture.to_dict()
src/api/routes/debug.py
+17
View File
@@ -0,0 +1,17 @@
"""
Debug Endpoints (Development Mode Only)
"""
from fastapi import APIRouter
from config.settings import settings
router = APIRouter()
@router.get("/config")
async def get_config():
"""Get current configuration"""
return {
"mode": settings.mode.value,
"storage": settings.storage_type.value,
"hardware_enabled": settings.enable_hardware
}
src/api/routes/devices.py
+21
View File
@@ -0,0 +1,21 @@
"""
Device Catalog Endpoints
"""
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from src.api.dependencies import get_db
router = APIRouter()
@router.get("/devices")
async def list_devices(db: Session = Depends(get_db)):
"""List all known devices"""
# TODO: Implement device listing
return {"devices": []}
@router.get("/devices/{device_id}")
async def get_device(device_id: int, db: Session = Depends(get_db)):
"""Get device details"""
# TODO: Implement device details
return {"device_id": device_id}
src/api/routes/hardware.py
+13
View File
@@ -0,0 +1,13 @@
"""
Hardware Control Endpoints (Development Mode Only)
"""
from fastapi import APIRouter
router = APIRouter()
@router.post("/hardware/scan")
async def start_scan():
"""Start RF scanning (Termux mode only)"""
# TODO: Implement T-Embed scanning
return {"status": "scanning"}
src/api/routes/query.py
+15
View File
@@ -0,0 +1,15 @@
"""
Query Endpoints
"""
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from src.api.dependencies import get_db
router = APIRouter()
@router.get("/query")
async def query_captures(db: Session = Depends(get_db)):
"""Query captures by location/time/protocol"""
# TODO: Implement geospatial queries
return {"captures": []}
src/api/routes/stats.py
+15
View File
@@ -0,0 +1,15 @@
"""
Statistics Endpoints
"""
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from src.api.dependencies import get_db
router = APIRouter()
@router.get("/stats")
async def get_stats(db: Session = Depends(get_db)):
"""Get platform statistics"""
# TODO: Implement statistics
return {"total_captures": 0}