feat: dataset export endpoint + SQLite dev-mode (geometry decoupling)

Dataset export (source data for model training):
- GET /api/v1/export?format=jsonl|csv|geojson with category/data_source
  filters; streams a labeled dataset (signal params + identified device +
  routed category) suitable for training a Sub-GHz classifier.

SQLite dev-mode (corrects FABLE brief: SQLite was NOT a drop-in swap):
- models.py made dialect-aware — JSONB->JSON, ARRAY(Text)->JSON, TSVECTOR
  ->Text via .with_variant(); PostGIS Geometry column + GiST index only
  defined when not on SQLite (lat/lon + haversine bbox used instead).
- config/database.py honors DATABASE_URL / a full-URL override and builds
  a SQLite engine (check_same_thread=False, no server pool) when the URL
  is sqlite; PostgreSQL keeps pooling + UTC session.

Verified: create_all + Capture/CaptureMatch/Device CRUD + JSON round-trip
+ bbox query all work on sqlite; postgres mode still defines geom + gist
index; 52/52 unit tests pass.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
2026-07-18 13:15:46 -07:00
parent 78c28fe43f
commit b1a3e2a11d
3 changed files with 171 additions and 30 deletions
+92 -2
View File
@@ -4,15 +4,17 @@ GigLez FastAPI Application - Simplified Version
Runs without database requirement for testing web interface
"""
import csv
import io
import json
import sys
from pathlib import Path
from typing import List
from typing import List, Optional
from datetime import datetime
from fastapi import FastAPI, Request, UploadFile, File, Form
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import HTMLResponse
from fastapi.responses import HTMLResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
@@ -240,6 +242,94 @@ async def get_stats():
}
# =============================================================================
# DATASET EXPORT (for model training)
# =============================================================================
# Columns exported for training datasets. Order is stable across formats.
EXPORT_FIELDS = [
"id", "filename", "frequency", "protocol", "preset",
"device_name", "device_category", "match_confidence", "match_method",
"latitude", "longitude", "gps_source", "timestamp",
"data_source", "session_id",
]
def _iter_export_rows(category: Optional[str], data_source: Optional[str]):
"""Yield capture dicts filtered by category / data_source."""
for cap in captures_storage:
if category and cap.get("device_category") != category:
continue
if data_source and cap.get("data_source") != data_source:
continue
yield cap
@app.get("/api/v1/export")
async def export_dataset(
format: str = "jsonl",
category: Optional[str] = None,
data_source: Optional[str] = None,
):
"""
Export captures as a labeled dataset for model training.
Query params:
- format: jsonl (default) | csv | geojson
- category: filter to one device category (e.g. "Weather Sensor")
- data_source: filter to one data source (e.g. "production")
Each row carries the signal parameters + the auto-identified device
label + category, suitable as training data for a Sub-GHz classifier.
Note: this JSON-backed store does not retain raw pulse timing arrays;
for RAW-pulse feature training, use the DB-backed path (Capture.raw_data).
"""
fmt = format.lower()
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
if fmt == "jsonl":
def gen():
for cap in _iter_export_rows(category, data_source):
yield json.dumps({k: cap.get(k) for k in EXPORT_FIELDS}) + "\n"
return StreamingResponse(
gen(),
media_type="application/x-ndjson",
headers={"Content-Disposition": f'attachment; filename="giglez_dataset_{ts}.jsonl"'},
)
if fmt == "csv":
def gen():
buf = io.StringIO()
writer = csv.DictWriter(buf, fieldnames=EXPORT_FIELDS, extrasaction="ignore")
writer.writeheader()
yield buf.getvalue()
for cap in _iter_export_rows(category, data_source):
buf.seek(0); buf.truncate(0)
writer.writerow({k: cap.get(k) for k in EXPORT_FIELDS})
yield buf.getvalue()
return StreamingResponse(
gen(),
media_type="text/csv",
headers={"Content-Disposition": f'attachment; filename="giglez_dataset_{ts}.csv"'},
)
if fmt == "geojson":
features = []
for cap in _iter_export_rows(category, data_source):
lat, lon = cap.get("latitude"), cap.get("longitude")
if lat is None or lon is None:
continue
features.append({
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [lon, lat]},
"properties": {k: cap.get(k) for k in EXPORT_FIELDS if k not in ("latitude", "longitude")},
})
return {"type": "FeatureCollection", "features": features}
return {"error": f"Unsupported format '{format}'. Use jsonl, csv, or geojson."}
@app.delete("/api/v1/admin/cleanup")
async def cleanup_test_data(data_source: str = "test"):
"""