Initial commit: Phase 1 & Phase 2 infrastructure complete
This commit is contained in:
@@ -0,0 +1,564 @@
|
||||
"""
|
||||
GigLez SQLAlchemy ORM Models
|
||||
|
||||
Database models matching the PostgreSQL + PostGIS schema
|
||||
Based on Wigle wardriving patterns adapted for IoT RF device mapping
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
from sqlalchemy import (
|
||||
Column, String, Integer, Float, DateTime, Text, Boolean,
|
||||
DECIMAL, ARRAY, ForeignKey, CheckConstraint, UniqueConstraint,
|
||||
Index, BYTEA
|
||||
)
|
||||
from sqlalchemy.orm import declarative_base, relationship
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from geoalchemy2 import Geometry
|
||||
from geoalchemy2.functions import ST_SetSRID, ST_MakePoint
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BASE
|
||||
# =============================================================================
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CORE MODELS
|
||||
# =============================================================================
|
||||
|
||||
class User(Base):
|
||||
"""User accounts for community features"""
|
||||
|
||||
__tablename__ = 'users'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Authentication
|
||||
username = Column(String(50), unique=True, nullable=False, index=True)
|
||||
email = Column(String(255), unique=True, nullable=False, index=True)
|
||||
password_hash = Column(String(255), nullable=False)
|
||||
|
||||
# Profile
|
||||
display_name = Column(String(100))
|
||||
avatar_url = Column(Text)
|
||||
bio = Column(Text)
|
||||
|
||||
# Statistics
|
||||
total_captures = Column(Integer, default=0)
|
||||
total_identifications = Column(Integer, default=0)
|
||||
reputation_score = Column(Integer, default=0)
|
||||
|
||||
# Settings
|
||||
api_key = Column(String(64), unique=True, index=True)
|
||||
email_verified = Column(Boolean, default=False)
|
||||
|
||||
# Timestamps
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
last_login = Column(DateTime)
|
||||
|
||||
# Relationships
|
||||
sessions = relationship("Session", back_populates="user")
|
||||
captures = relationship("Capture", back_populates="user")
|
||||
identifications = relationship("Identification", back_populates="user")
|
||||
votes = relationship("Vote", back_populates="user")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<User(id={self.id}, username='{self.username}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'username': self.username,
|
||||
'display_name': self.display_name,
|
||||
'total_captures': self.total_captures,
|
||||
'reputation_score': self.reputation_score,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None
|
||||
}
|
||||
|
||||
|
||||
class Session(Base):
|
||||
"""Wardriving/capture sessions (Wigle pattern)"""
|
||||
|
||||
__tablename__ = 'sessions'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
user_id = Column(Integer, ForeignKey('users.id', ondelete='SET NULL'), index=True)
|
||||
|
||||
# Session Metadata
|
||||
session_uuid = Column(String(64), unique=True, nullable=False, index=True)
|
||||
name = Column(String(200))
|
||||
description = Column(Text)
|
||||
started_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
|
||||
ended_at = Column(DateTime)
|
||||
|
||||
# Privacy Settings
|
||||
is_public = Column(Boolean, default=True, index=True)
|
||||
anonymize_gps = Column(Boolean, default=False)
|
||||
gps_precision_meters = Column(Integer, default=10)
|
||||
|
||||
# Session Statistics (auto-updated by triggers)
|
||||
total_captures = Column(Integer, default=0)
|
||||
unique_devices = Column(Integer, default=0)
|
||||
distance_km = Column(DECIMAL(10, 2))
|
||||
|
||||
# Bounding Box (auto-updated by triggers)
|
||||
min_latitude = Column(DECIMAL(10, 8))
|
||||
max_latitude = Column(DECIMAL(10, 8))
|
||||
min_longitude = Column(DECIMAL(11, 8))
|
||||
max_longitude = Column(DECIMAL(11, 8))
|
||||
|
||||
# Relationships
|
||||
user = relationship("User", back_populates="sessions")
|
||||
captures = relationship("Capture", back_populates="session")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Session(id={self.id}, uuid='{self.session_uuid}', name='{self.name}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'session_uuid': self.session_uuid,
|
||||
'name': self.name,
|
||||
'started_at': self.started_at.isoformat() if self.started_at else None,
|
||||
'ended_at': self.ended_at.isoformat() if self.ended_at else None,
|
||||
'total_captures': self.total_captures,
|
||||
'unique_devices': self.unique_devices,
|
||||
'is_public': self.is_public
|
||||
}
|
||||
|
||||
|
||||
class Device(Base):
|
||||
"""Known IoT device types from signature databases"""
|
||||
|
||||
__tablename__ = 'devices'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Device Identification
|
||||
manufacturer = Column(String(200), index=True)
|
||||
model = Column(String(200))
|
||||
device_type = Column(String(100), index=True)
|
||||
description = Column(Text)
|
||||
|
||||
# RF Characteristics
|
||||
typical_frequency = Column(Integer, index=True)
|
||||
frequency_range_low = Column(Integer)
|
||||
frequency_range_high = Column(Integer)
|
||||
modulation_types = Column(ARRAY(Text))
|
||||
|
||||
# Protocol Information
|
||||
protocol = Column(String(100), index=True)
|
||||
bit_length = Column(Integer)
|
||||
encoding = Column(String(50))
|
||||
|
||||
# Metadata
|
||||
fcc_id = Column(String(50))
|
||||
manufacturer_code = Column(String(50))
|
||||
|
||||
# Community Data
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
verified = Column(Boolean, default=False, index=True)
|
||||
verification_count = Column(Integer, default=0)
|
||||
|
||||
# Source
|
||||
source = Column(String(50), index=True) # 'flipper', 'rtl433', 'urh', 'community'
|
||||
source_url = Column(Text)
|
||||
|
||||
# Full-text search (auto-updated by trigger)
|
||||
search_vector = Column('search_vector', nullable=True)
|
||||
|
||||
# Relationships
|
||||
signatures = relationship("Signature", back_populates="device")
|
||||
captures = relationship("Capture", back_populates="device")
|
||||
capture_matches = relationship("CaptureMatch", back_populates="device")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Device(id={self.id}, manufacturer='{self.manufacturer}', model='{self.model}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'manufacturer': self.manufacturer,
|
||||
'model': self.model,
|
||||
'device_type': self.device_type,
|
||||
'description': self.description,
|
||||
'typical_frequency': self.typical_frequency,
|
||||
'protocol': self.protocol,
|
||||
'verified': self.verified,
|
||||
'source': self.source
|
||||
}
|
||||
|
||||
|
||||
class Capture(Base):
|
||||
"""RF signal captures with GPS coordinates"""
|
||||
|
||||
__tablename__ = 'captures'
|
||||
|
||||
# Primary Key: SHA256 hash of file (Wigle deduplication pattern)
|
||||
file_hash = Column(String(64), primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
session_id = Column(Integer, ForeignKey('sessions.id', ondelete='CASCADE'), index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id', ondelete='SET NULL'), index=True)
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='SET NULL'), index=True)
|
||||
|
||||
# GPS Data
|
||||
latitude = Column(DECIMAL(10, 8), nullable=False)
|
||||
longitude = Column(DECIMAL(11, 8), nullable=False)
|
||||
altitude = Column(DECIMAL(8, 2))
|
||||
gps_accuracy = Column(DECIMAL(6, 2))
|
||||
geom = Column(Geometry('POINT', srid=4326)) # Auto-populated by trigger
|
||||
|
||||
# Timestamps
|
||||
captured_at = Column(DateTime, nullable=False, index=True)
|
||||
uploaded_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
|
||||
# RF Signal Data
|
||||
frequency = Column(Integer, nullable=False, index=True)
|
||||
rssi = Column(Integer)
|
||||
modulation = Column(String(50))
|
||||
preset = Column(String(100))
|
||||
|
||||
# Protocol Information (if decoded)
|
||||
protocol = Column(String(100), index=True)
|
||||
bit_length = Column(Integer)
|
||||
key_data = Column(BYTEA)
|
||||
timing_element = Column(Integer)
|
||||
|
||||
# Raw Signal Data
|
||||
raw_data = Column(Text)
|
||||
raw_format = Column(String(20)) # 'RAW', 'BinRAW', 'KEY'
|
||||
|
||||
# File Storage
|
||||
file_path = Column(String(500))
|
||||
file_size = Column(Integer)
|
||||
|
||||
# Automatic Matching Results (best match)
|
||||
match_confidence = Column(DECIMAL(5, 4))
|
||||
match_method = Column(String(50))
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
CheckConstraint('latitude >= -90 AND latitude <= 90', name='valid_latitude'),
|
||||
CheckConstraint('longitude >= -180 AND longitude <= 180', name='valid_longitude'),
|
||||
CheckConstraint('match_confidence >= 0 AND match_confidence <= 1', name='valid_confidence'),
|
||||
CheckConstraint('frequency >= 300000000 AND frequency <= 928000000', name='valid_frequency'),
|
||||
Index('idx_captures_geom', 'geom', postgresql_using='gist'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
session = relationship("Session", back_populates="captures")
|
||||
user = relationship("User", back_populates="captures")
|
||||
device = relationship("Device", back_populates="captures")
|
||||
capture_matches = relationship("CaptureMatch", back_populates="capture")
|
||||
identifications = relationship("Identification", back_populates="capture")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Capture(hash='{self.file_hash[:8]}...', freq={self.frequency}, protocol='{self.protocol}')>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'file_hash': self.file_hash,
|
||||
'latitude': float(self.latitude) if self.latitude else None,
|
||||
'longitude': float(self.longitude) if self.longitude else None,
|
||||
'altitude': float(self.altitude) if self.altitude else None,
|
||||
'gps_accuracy': float(self.gps_accuracy) if self.gps_accuracy else None,
|
||||
'captured_at': self.captured_at.isoformat() if self.captured_at else None,
|
||||
'frequency': self.frequency,
|
||||
'rssi': self.rssi,
|
||||
'modulation': self.modulation,
|
||||
'protocol': self.protocol,
|
||||
'bit_length': self.bit_length,
|
||||
'device_id': self.device_id,
|
||||
'match_confidence': float(self.match_confidence) if self.match_confidence else None,
|
||||
'match_method': self.match_method
|
||||
}
|
||||
|
||||
|
||||
class Signature(Base):
|
||||
"""Protocol signatures for device matching"""
|
||||
|
||||
__tablename__ = 'signatures'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='CASCADE'), index=True)
|
||||
|
||||
# Signature Pattern
|
||||
protocol = Column(String(100), nullable=False, index=True)
|
||||
frequency = Column(Integer, index=True)
|
||||
modulation = Column(String(50))
|
||||
|
||||
# Matching Criteria
|
||||
bit_pattern = Column(BYTEA)
|
||||
bit_mask = Column(BYTEA)
|
||||
timing_min = Column(Integer)
|
||||
timing_max = Column(Integer)
|
||||
|
||||
# Raw Pattern
|
||||
pattern_regex = Column(Text)
|
||||
|
||||
# Confidence Weighting
|
||||
weight = Column(DECIMAL(4, 3), default=1.0)
|
||||
|
||||
# Metadata
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
source = Column(String(50), index=True)
|
||||
source_file = Column(String(500))
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
UniqueConstraint('device_id', 'protocol', 'bit_pattern', name='uq_device_protocol_pattern'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
device = relationship("Device", back_populates="signatures")
|
||||
flipper_signature = relationship("FlipperSignature", back_populates="signature", uselist=False)
|
||||
rtl433_protocol = relationship("RTL433Protocol", back_populates="signature", uselist=False)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Signature(id={self.id}, protocol='{self.protocol}', device_id={self.device_id})>"
|
||||
|
||||
|
||||
class CaptureMatch(Base):
|
||||
"""Many-to-many mapping of captures to devices with confidence scores"""
|
||||
|
||||
__tablename__ = 'capture_matches'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
capture_id = Column(String(64), ForeignKey('captures.file_hash', ondelete='CASCADE'), index=True)
|
||||
device_id = Column(Integer, ForeignKey('devices.id', ondelete='CASCADE'), index=True)
|
||||
signature_id = Column(Integer, ForeignKey('signatures.id', ondelete='SET NULL'), index=True)
|
||||
|
||||
# Match Details
|
||||
confidence = Column(DECIMAL(5, 4), nullable=False, index=True)
|
||||
match_method = Column(String(50), nullable=False)
|
||||
match_details = Column(JSONB)
|
||||
|
||||
# Timestamp
|
||||
matched_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
CheckConstraint('confidence >= 0 AND confidence <= 1', name='valid_match_confidence'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
capture = relationship("Capture", back_populates="capture_matches")
|
||||
device = relationship("Device", back_populates="capture_matches")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<CaptureMatch(capture='{self.capture_id[:8]}...', device={self.device_id}, confidence={self.confidence})>"
|
||||
|
||||
|
||||
class Identification(Base):
|
||||
"""User-submitted device identifications"""
|
||||
|
||||
__tablename__ = 'identifications'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
capture_id = Column(String(64), ForeignKey('captures.file_hash', ondelete='CASCADE'), index=True)
|
||||
device_id = Column(Integer, ForeignKey('devices.id'), index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id'), index=True)
|
||||
|
||||
# Identification Details
|
||||
confidence = Column(String(20)) # 'certain', 'likely', 'guess'
|
||||
notes = Column(Text)
|
||||
|
||||
# Visual Evidence
|
||||
photo_urls = Column(ARRAY(Text))
|
||||
|
||||
# Community Validation
|
||||
upvotes = Column(Integer, default=0)
|
||||
downvotes = Column(Integer, default=0)
|
||||
verified = Column(Boolean, default=False, index=True)
|
||||
verified_by = Column(Integer, ForeignKey('users.id'))
|
||||
verified_at = Column(DateTime)
|
||||
|
||||
# Timestamp
|
||||
submitted_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
UniqueConstraint('capture_id', 'user_id', 'device_id', name='uq_capture_user_device'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
capture = relationship("Capture", back_populates="identifications")
|
||||
user = relationship("User", back_populates="identifications", foreign_keys=[user_id])
|
||||
votes = relationship("Vote", back_populates="identification")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Identification(id={self.id}, capture='{self.capture_id[:8]}...', device={self.device_id})>"
|
||||
|
||||
|
||||
class Vote(Base):
|
||||
"""Votes on device identifications"""
|
||||
|
||||
__tablename__ = 'votes'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Keys
|
||||
identification_id = Column(Integer, ForeignKey('identifications.id', ondelete='CASCADE'), index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id'), index=True)
|
||||
|
||||
# Vote Data
|
||||
vote_type = Column(Integer, nullable=False) # 1 = upvote, -1 = downvote
|
||||
voted_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Constraints
|
||||
__table_args__ = (
|
||||
CheckConstraint('vote_type IN (1, -1)', name='valid_vote'),
|
||||
UniqueConstraint('identification_id', 'user_id', name='uq_identification_user'),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
identification = relationship("Identification", back_populates="votes")
|
||||
user = relationship("User", back_populates="votes")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Vote(id={self.id}, identification={self.identification_id}, type={self.vote_type})>"
|
||||
|
||||
|
||||
class UploadMarker(Base):
|
||||
"""Track last uploaded capture per user/session (Wigle incremental sync pattern)"""
|
||||
|
||||
__tablename__ = 'upload_markers'
|
||||
|
||||
# Composite Primary Key
|
||||
user_id = Column(Integer, ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
|
||||
session_id = Column(Integer, ForeignKey('sessions.id', ondelete='CASCADE'), primary_key=True)
|
||||
|
||||
# Marker Data
|
||||
last_capture_hash = Column(String(64))
|
||||
last_upload_time = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<UploadMarker(user={self.user_id}, session={self.session_id})>"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# SIGNATURE DATABASE MODELS
|
||||
# =============================================================================
|
||||
|
||||
class FlipperSignature(Base):
|
||||
"""Flipper Zero specific signature data"""
|
||||
|
||||
__tablename__ = 'flipper_signatures'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Key
|
||||
signature_id = Column(Integer, ForeignKey('signatures.id', ondelete='CASCADE'), index=True)
|
||||
|
||||
# Flipper-Specific Fields
|
||||
filetype = Column(String(50))
|
||||
version = Column(Integer)
|
||||
preset = Column(String(100), index=True)
|
||||
|
||||
# Custom Preset Data
|
||||
custom_preset_module = Column(String(50))
|
||||
custom_preset_data = Column(BYTEA)
|
||||
|
||||
# Protocol Data
|
||||
protocol = Column(String(100), index=True)
|
||||
bit = Column(Integer)
|
||||
key = Column(BYTEA)
|
||||
te = Column(Integer)
|
||||
|
||||
# RAW Data
|
||||
raw_data = Column(Text)
|
||||
bin_raw_bit = Column(Integer)
|
||||
bin_raw_te = Column(Integer)
|
||||
bin_raw_data = Column(BYTEA)
|
||||
|
||||
# Source
|
||||
source_file = Column(String(500))
|
||||
imported_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
signature = relationship("Signature", back_populates="flipper_signature")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<FlipperSignature(id={self.id}, protocol='{self.protocol}')>"
|
||||
|
||||
|
||||
class RTL433Protocol(Base):
|
||||
"""RTL_433 specific protocol data"""
|
||||
|
||||
__tablename__ = 'rtl433_protocols'
|
||||
|
||||
# Primary Key
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
# Foreign Key
|
||||
signature_id = Column(Integer, ForeignKey('signatures.id', ondelete='CASCADE'), index=True)
|
||||
|
||||
# Protocol Identification
|
||||
protocol_number = Column(Integer, index=True)
|
||||
protocol_name = Column(String(200))
|
||||
model = Column(String(200), index=True)
|
||||
|
||||
# RF Characteristics
|
||||
frequency = Column(Integer)
|
||||
modulation = Column(String(50), index=True)
|
||||
|
||||
# Timing Information
|
||||
short_width = Column(Integer)
|
||||
long_width = Column(Integer)
|
||||
reset_limit = Column(Integer)
|
||||
gap_limit = Column(Integer)
|
||||
|
||||
# Decoding
|
||||
decoder_type = Column(String(50))
|
||||
bit_count = Column(Integer)
|
||||
|
||||
# JSON Fields Mapping
|
||||
json_fields = Column(JSONB)
|
||||
|
||||
# Source
|
||||
source_file = Column(String(500))
|
||||
imported_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
signature = relationship("Signature", back_populates="rtl433_protocol")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<RTL433Protocol(id={self.id}, name='{self.protocol_name}')>"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# HELPER FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
def create_all_tables(engine):
|
||||
"""Create all tables in the database"""
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
def drop_all_tables(engine):
|
||||
"""Drop all tables from the database (use with caution!)"""
|
||||
Base.metadata.drop_all(engine)
|
||||
Reference in New Issue
Block a user