feat: dataset export endpoint + SQLite dev-mode (geometry decoupling)

Dataset export (source data for model training):
- GET /api/v1/export?format=jsonl|csv|geojson with category/data_source
  filters; streams a labeled dataset (signal params + identified device +
  routed category) suitable for training a Sub-GHz classifier.

SQLite dev-mode (corrects FABLE brief: SQLite was NOT a drop-in swap):
- models.py made dialect-aware — JSONB->JSON, ARRAY(Text)->JSON, TSVECTOR
  ->Text via .with_variant(); PostGIS Geometry column + GiST index only
  defined when not on SQLite (lat/lon + haversine bbox used instead).
- config/database.py honors DATABASE_URL / a full-URL override and builds
  a SQLite engine (check_same_thread=False, no server pool) when the URL
  is sqlite; PostgreSQL keeps pooling + UTC session.

Verified: create_all + Capture/CaptureMatch/Device CRUD + JSON round-trip
+ bbox query all work on sqlite; postgres mode still defines geom + gist
index; 52/52 unit tests pass.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
2026-07-18 13:15:46 -07:00
parent 78c28fe43f
commit b1a3e2a11d
3 changed files with 171 additions and 30 deletions
+41 -16
View File
@@ -24,7 +24,8 @@ class DatabaseConfig:
password: str = "giglez_secure_password_2026",
pool_size: int = 10,
max_overflow: int = 20,
echo: bool = False
echo: bool = False,
url: Optional[str] = None
):
"""
Initialize database configuration
@@ -38,6 +39,9 @@ class DatabaseConfig:
pool_size: Connection pool size (Wigle pattern: moderate pooling)
max_overflow: Max overflow connections
echo: Echo SQL queries (debug mode)
url: Full SQLAlchemy URL override (e.g. sqlite:///./giglez.db).
When set, it takes precedence over the host/port/user fields.
Falls back to the DATABASE_URL env var.
"""
self.host = host
self.port = port
@@ -47,40 +51,61 @@ class DatabaseConfig:
self.pool_size = pool_size
self.max_overflow = max_overflow
self.echo = echo
self.url = url or os.getenv("DATABASE_URL")
self._engine: Optional = None
self._session_factory: Optional[sessionmaker] = None
@property
def is_sqlite(self) -> bool:
return bool(self.url) and self.url.startswith("sqlite")
@property
def connection_string(self) -> str:
"""Generate PostgreSQL connection string"""
"""SQLAlchemy connection string (URL override wins, else PostgreSQL)"""
if self.url:
return self.url
return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}"
@property
def connection_string_safe(self) -> str:
"""Generate connection string without password (for logging)"""
"""Connection string without password (for logging)"""
if self.url:
# sqlite URLs carry no password; postgres URLs would — mask them
if self.is_sqlite:
return self.url
return "postgresql://****"
return f"postgresql://{self.user}:****@{self.host}:{self.port}/{self.database}"
def get_engine(self):
"""
Get or create SQLAlchemy engine
Get or create SQLAlchemy engine.
Uses connection pooling for performance (Wigle pattern)
PostgreSQL uses connection pooling + UTC session (Wigle pattern).
SQLite (dev/MVP) uses a simple engine with cross-thread access enabled.
"""
if self._engine is None:
logger.info(f"Creating database engine: {self.connection_string_safe}")
self._engine = create_engine(
self.connection_string,
poolclass=QueuePool,
pool_size=self.pool_size,
max_overflow=self.max_overflow,
pool_pre_ping=True, # Verify connections before using
echo=self.echo,
connect_args={
"options": "-c timezone=utc" # Always use UTC
}
)
if self.is_sqlite:
# SQLite dev mode: no server-side pool, allow use across threads
self._engine = create_engine(
self.connection_string,
echo=self.echo,
connect_args={"check_same_thread": False},
)
else:
self._engine = create_engine(
self.connection_string,
poolclass=QueuePool,
pool_size=self.pool_size,
max_overflow=self.max_overflow,
pool_pre_ping=True, # Verify connections before using
echo=self.echo,
connect_args={
"options": "-c timezone=utc" # Always use UTC
}
)
logger.success("Database engine created successfully")