diff --git a/config/database.py b/config/database.py index 503be54..c2627b9 100644 --- a/config/database.py +++ b/config/database.py @@ -24,7 +24,8 @@ class DatabaseConfig: password: str = "giglez_secure_password_2026", pool_size: int = 10, max_overflow: int = 20, - echo: bool = False + echo: bool = False, + url: Optional[str] = None ): """ Initialize database configuration @@ -38,6 +39,9 @@ class DatabaseConfig: pool_size: Connection pool size (Wigle pattern: moderate pooling) max_overflow: Max overflow connections echo: Echo SQL queries (debug mode) + url: Full SQLAlchemy URL override (e.g. sqlite:///./giglez.db). + When set, it takes precedence over the host/port/user fields. + Falls back to the DATABASE_URL env var. """ self.host = host self.port = port @@ -47,40 +51,61 @@ class DatabaseConfig: self.pool_size = pool_size self.max_overflow = max_overflow self.echo = echo + self.url = url or os.getenv("DATABASE_URL") self._engine: Optional = None self._session_factory: Optional[sessionmaker] = None + @property + def is_sqlite(self) -> bool: + return bool(self.url) and self.url.startswith("sqlite") + @property def connection_string(self) -> str: - """Generate PostgreSQL connection string""" + """SQLAlchemy connection string (URL override wins, else PostgreSQL)""" + if self.url: + return self.url return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}" @property def connection_string_safe(self) -> str: - """Generate connection string without password (for logging)""" + """Connection string without password (for logging)""" + if self.url: + # sqlite URLs carry no password; postgres URLs would — mask them + if self.is_sqlite: + return self.url + return "postgresql://****" return f"postgresql://{self.user}:****@{self.host}:{self.port}/{self.database}" def get_engine(self): """ - Get or create SQLAlchemy engine + Get or create SQLAlchemy engine. - Uses connection pooling for performance (Wigle pattern) + PostgreSQL uses connection pooling + UTC session (Wigle pattern). + SQLite (dev/MVP) uses a simple engine with cross-thread access enabled. """ if self._engine is None: logger.info(f"Creating database engine: {self.connection_string_safe}") - self._engine = create_engine( - self.connection_string, - poolclass=QueuePool, - pool_size=self.pool_size, - max_overflow=self.max_overflow, - pool_pre_ping=True, # Verify connections before using - echo=self.echo, - connect_args={ - "options": "-c timezone=utc" # Always use UTC - } - ) + if self.is_sqlite: + # SQLite dev mode: no server-side pool, allow use across threads + self._engine = create_engine( + self.connection_string, + echo=self.echo, + connect_args={"check_same_thread": False}, + ) + else: + self._engine = create_engine( + self.connection_string, + poolclass=QueuePool, + pool_size=self.pool_size, + max_overflow=self.max_overflow, + pool_pre_ping=True, # Verify connections before using + echo=self.echo, + connect_args={ + "options": "-c timezone=utc" # Always use UTC + } + ) logger.success("Database engine created successfully") diff --git a/src/api/main_simple.py b/src/api/main_simple.py index 1ccc1d2..9d59465 100644 --- a/src/api/main_simple.py +++ b/src/api/main_simple.py @@ -4,15 +4,17 @@ GigLez FastAPI Application - Simplified Version Runs without database requirement for testing web interface """ +import csv +import io import json import sys from pathlib import Path -from typing import List +from typing import List, Optional from datetime import datetime from fastapi import FastAPI, Request, UploadFile, File, Form from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -240,6 +242,94 @@ async def get_stats(): } +# ============================================================================= +# DATASET EXPORT (for model training) +# ============================================================================= + +# Columns exported for training datasets. Order is stable across formats. +EXPORT_FIELDS = [ + "id", "filename", "frequency", "protocol", "preset", + "device_name", "device_category", "match_confidence", "match_method", + "latitude", "longitude", "gps_source", "timestamp", + "data_source", "session_id", +] + + +def _iter_export_rows(category: Optional[str], data_source: Optional[str]): + """Yield capture dicts filtered by category / data_source.""" + for cap in captures_storage: + if category and cap.get("device_category") != category: + continue + if data_source and cap.get("data_source") != data_source: + continue + yield cap + + +@app.get("/api/v1/export") +async def export_dataset( + format: str = "jsonl", + category: Optional[str] = None, + data_source: Optional[str] = None, +): + """ + Export captures as a labeled dataset for model training. + + Query params: + - format: jsonl (default) | csv | geojson + - category: filter to one device category (e.g. "Weather Sensor") + - data_source: filter to one data source (e.g. "production") + + Each row carries the signal parameters + the auto-identified device + label + category, suitable as training data for a Sub-GHz classifier. + + Note: this JSON-backed store does not retain raw pulse timing arrays; + for RAW-pulse feature training, use the DB-backed path (Capture.raw_data). + """ + fmt = format.lower() + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + + if fmt == "jsonl": + def gen(): + for cap in _iter_export_rows(category, data_source): + yield json.dumps({k: cap.get(k) for k in EXPORT_FIELDS}) + "\n" + return StreamingResponse( + gen(), + media_type="application/x-ndjson", + headers={"Content-Disposition": f'attachment; filename="giglez_dataset_{ts}.jsonl"'}, + ) + + if fmt == "csv": + def gen(): + buf = io.StringIO() + writer = csv.DictWriter(buf, fieldnames=EXPORT_FIELDS, extrasaction="ignore") + writer.writeheader() + yield buf.getvalue() + for cap in _iter_export_rows(category, data_source): + buf.seek(0); buf.truncate(0) + writer.writerow({k: cap.get(k) for k in EXPORT_FIELDS}) + yield buf.getvalue() + return StreamingResponse( + gen(), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="giglez_dataset_{ts}.csv"'}, + ) + + if fmt == "geojson": + features = [] + for cap in _iter_export_rows(category, data_source): + lat, lon = cap.get("latitude"), cap.get("longitude") + if lat is None or lon is None: + continue + features.append({ + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [lon, lat]}, + "properties": {k: cap.get(k) for k in EXPORT_FIELDS if k not in ("latitude", "longitude")}, + }) + return {"type": "FeatureCollection", "features": features} + + return {"error": f"Unsupported format '{format}'. Use jsonl, csv, or geojson."} + + @app.delete("/api/v1/admin/cleanup") async def cleanup_test_data(data_source: str = "test"): """ diff --git a/src/database/models.py b/src/database/models.py index 536f277..1bb9775 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -5,17 +5,38 @@ Database models matching the PostgreSQL + PostGIS schema Based on Wigle wardriving patterns adapted for IoT RF device mapping """ +import os from datetime import datetime from typing import Optional, List from sqlalchemy import ( Column, String, Integer, Float, DateTime, Text, Boolean, DECIMAL, ARRAY, ForeignKey, CheckConstraint, UniqueConstraint, - Index, LargeBinary + Index, LargeBinary, JSON ) from sqlalchemy.orm import declarative_base, relationship -from sqlalchemy.dialects.postgresql import JSONB -from geoalchemy2 import Geometry -from geoalchemy2.functions import ST_SetSRID, ST_MakePoint +from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR + +# ============================================================================= +# DIALECT AWARENESS (PostgreSQL/PostGIS in prod, SQLite in dev) +# ============================================================================= +# The production schema targets PostgreSQL + PostGIS, but dev/MVP must run on +# SQLite with no external services. We keep PostgreSQL-native behaviour on +# postgres via `.with_variant()` while degrading gracefully on SQLite. + +_DATABASE_URL = os.getenv("DATABASE_URL", "") +IS_SQLITE = _DATABASE_URL.startswith("sqlite") + +# JSON: JSONB on postgres, generic JSON (stored as TEXT) on sqlite +JSON_TYPE = JSON().with_variant(JSONB(), "postgresql") +# Text arrays: native ARRAY on postgres, JSON list on sqlite +STR_ARRAY_TYPE = JSON().with_variant(ARRAY(Text), "postgresql") +# Full-text search vector: TSVECTOR on postgres (trigger-populated), Text on sqlite +SEARCH_VECTOR_TYPE = Text().with_variant(TSVECTOR(), "postgresql") + +if not IS_SQLITE: + # geoalchemy2 is only importable/usable against a spatial backend + from geoalchemy2 import Geometry + from geoalchemy2.functions import ST_SetSRID, ST_MakePoint # ============================================================================= @@ -154,7 +175,7 @@ class Device(Base): typical_frequency = Column(Integer, index=True) frequency_range_low = Column(Integer) frequency_range_high = Column(Integer) - modulation_types = Column(ARRAY(Text)) + modulation_types = Column(STR_ARRAY_TYPE) # Protocol Information protocol = Column(String(100), index=True) @@ -174,8 +195,8 @@ class Device(Base): source = Column(String(50), index=True) # 'flipper', 'rtl433', 'urh', 'community' source_url = Column(Text) - # Full-text search (auto-updated by trigger) - search_vector = Column('search_vector', nullable=True) + # Full-text search (auto-updated by trigger on postgres) + search_vector = Column('search_vector', SEARCH_VECTOR_TYPE, nullable=True) # Relationships signatures = relationship("Signature", back_populates="device") @@ -218,7 +239,9 @@ class Capture(Base): longitude = Column(DECIMAL(11, 8), nullable=False) altitude = Column(DECIMAL(8, 2)) gps_accuracy = Column(DECIMAL(6, 2)) - geom = Column(Geometry('POINT', srid=4326)) # Auto-populated by trigger + # Spatial geometry only exists on PostGIS; SQLite uses lat/lon + haversine + if not IS_SQLITE: + geom = Column(Geometry('POINT', srid=4326)) # Auto-populated by trigger # Timestamps captured_at = Column(DateTime, nullable=False, index=True) @@ -254,7 +277,10 @@ class Capture(Base): CheckConstraint('longitude >= -180 AND longitude <= 180', name='valid_longitude'), CheckConstraint('match_confidence >= 0 AND match_confidence <= 1', name='valid_confidence'), CheckConstraint('frequency >= 300000000 AND frequency <= 928000000', name='valid_frequency'), - Index('idx_captures_geom', 'geom', postgresql_using='gist'), + ) + ( + # GiST spatial index only applies to the PostGIS geometry column + (Index('idx_captures_geom', 'geom', postgresql_using='gist'),) + if not IS_SQLITE else () ) # Relationships @@ -350,7 +376,7 @@ class CaptureMatch(Base): # Match Details confidence = Column(DECIMAL(5, 4), nullable=False, index=True) match_method = Column(String(50), nullable=False) - match_details = Column(JSONB) + match_details = Column(JSON_TYPE) # Timestamp matched_at = Column(DateTime, default=datetime.utcnow) @@ -386,7 +412,7 @@ class Identification(Base): notes = Column(Text) # Visual Evidence - photo_urls = Column(ARRAY(Text)) + photo_urls = Column(STR_ARRAY_TYPE) # Community Validation upvotes = Column(Integer, default=0) @@ -540,7 +566,7 @@ class RTL433Protocol(Base): bit_count = Column(Integer) # JSON Fields Mapping - json_fields = Column(JSONB) + json_fields = Column(JSON_TYPE) # Source source_file = Column(String(500))