Phase 1 & 2: Cleanup redundant code and integrate pattern decoder
## Phase 1: Code Cleanup (~1,859 lines removed) **Deleted Redundant Matchers:** - ❌ strategies_orm.py (356 lines) - Old ORM-based strategies - ❌ simple_matcher.py (351 lines) - Replaced by strategies.py - ❌ rtl433_matcher.py (352 lines) - Replaced by strategies.py **Archived Old Scripts:** - Moved 11 one-time analysis/import scripts to scripts/archive/ - Scripts: analyze_flipper_signatures, analyze_tembed_files, identify_tembed_devices, import_flipper_sqlite, import_tembed_signatures, match_tembed_with_db, match_with_flipper_db, rematch_captures, test_gps_extraction, test_tembed_matching, test_wardriving_import **Consolidated API:** - Renamed main.py → main_orm_legacy.py (archived old ORM-based API) - main_simple.py is now the primary production API ## Phase 2: Pattern Decoder Integration ✅ **CRITICAL FIX: Pattern decoder now integrated into production API!** **Changes:** 1. Updated main_simple.py to use unified SignatureMatcher 2. Added 6 strategies to matcher pipeline: - ExactMatcher (protocol + frequency) - FrequencyMatcher (frequency-based) - BitPatternMatcher (data patterns) - TimingMatcher (timing-based) - RTL433DecoderStrategy (RTL_433 decoder) - PatternBasedStrategy (NEW! Pattern decoder for short captures) 3. Created MockDB class for simplified mode (no real database) 4. Replaced old get_matcher() with get_matcher_engine() 5. Updated upload handler to use MatchResult format 6. All matches now include confidence scores and match methods **Result:** - Pattern decoder is NOW ACTIVE in production 🎉 - Unified matching pipeline with 6 strategies - Cleaner codebase (-1,859 lines) - Single source of truth for matching logic 🎉 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+156
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Analyze Flipper Zero signature database
|
||||
|
||||
Parses all Flipper Zero .sub files and creates a comprehensive device signature database
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
|
||||
|
||||
def analyze_flipper_database(flipper_dir: Path):
|
||||
"""Analyze all Flipper Zero .sub files"""
|
||||
|
||||
print("="*80)
|
||||
print("FLIPPER ZERO SIGNATURE DATABASE ANALYSIS")
|
||||
print("="*80)
|
||||
print()
|
||||
|
||||
parser = SubFileParser()
|
||||
|
||||
# Find all .sub files
|
||||
sub_files = list(flipper_dir.glob('**/*.sub'))
|
||||
print(f"Found {len(sub_files)} Flipper Zero .sub files\n")
|
||||
|
||||
# Parse all files
|
||||
signatures = []
|
||||
parse_errors = []
|
||||
|
||||
for sub_file in sub_files:
|
||||
try:
|
||||
metadata = parser.parse(str(sub_file))
|
||||
|
||||
# Extract device name from path
|
||||
device_name = sub_file.stem
|
||||
|
||||
signatures.append({
|
||||
'filename': sub_file.name,
|
||||
'device_name': device_name,
|
||||
'path': str(sub_file.relative_to(flipper_dir)),
|
||||
'frequency': metadata.frequency,
|
||||
'protocol': metadata.protocol,
|
||||
'file_format': metadata.file_format,
|
||||
'modulation': metadata.modulation,
|
||||
'bit_length': metadata.bit_length,
|
||||
'has_raw_data': bool(metadata.raw_data),
|
||||
'raw_samples': len(metadata.raw_data) if metadata.raw_data else 0
|
||||
})
|
||||
except Exception as e:
|
||||
parse_errors.append((sub_file.name, str(e)))
|
||||
|
||||
print(f"Successfully parsed: {len(signatures)} files")
|
||||
print(f"Parse errors: {len(parse_errors)} files\n")
|
||||
|
||||
# Analyze by frequency
|
||||
print("-"*80)
|
||||
print("FREQUENCY DISTRIBUTION")
|
||||
print("-"*80)
|
||||
|
||||
freq_groups = defaultdict(list)
|
||||
for sig in signatures:
|
||||
freq_mhz = sig['frequency'] / 1e6 if sig['frequency'] else 0
|
||||
freq_groups[freq_mhz].append(sig)
|
||||
|
||||
for freq in sorted(freq_groups.keys()):
|
||||
if freq > 0:
|
||||
count = len(freq_groups[freq])
|
||||
print(f"{freq:8.2f} MHz: {count:3d} devices")
|
||||
|
||||
# Analyze by protocol
|
||||
print(f"\n{'-'*80}")
|
||||
print("PROTOCOL DISTRIBUTION")
|
||||
print("-"*80)
|
||||
|
||||
protocol_groups = defaultdict(list)
|
||||
for sig in signatures:
|
||||
proto = sig['protocol'] or 'RAW'
|
||||
protocol_groups[proto].append(sig)
|
||||
|
||||
for proto in sorted(protocol_groups.keys(), key=lambda x: len(protocol_groups[x]), reverse=True)[:15]:
|
||||
count = len(protocol_groups[proto])
|
||||
print(f"{proto:30s}: {count:3d} devices")
|
||||
|
||||
# Analyze by format
|
||||
print(f"\n{'-'*80}")
|
||||
print("FILE FORMAT DISTRIBUTION")
|
||||
print("-"*80)
|
||||
|
||||
format_groups = defaultdict(list)
|
||||
for sig in signatures:
|
||||
format_groups[sig['file_format']].append(sig)
|
||||
|
||||
for fmt in sorted(format_groups.keys()):
|
||||
count = len(format_groups[fmt])
|
||||
print(f"{fmt:10s}: {count:3d} files")
|
||||
|
||||
# Show sample devices by frequency band
|
||||
print(f"\n{'-'*80}")
|
||||
print("SAMPLE DEVICES BY FREQUENCY BAND")
|
||||
print("-"*80)
|
||||
|
||||
# Group into common RF bands
|
||||
bands = {
|
||||
'300-350 MHz (Garage/Gate)': (300, 350),
|
||||
'400-450 MHz (Key Fobs/Remotes)': (400, 450),
|
||||
'800-900 MHz (Sensors/Utility)': (800, 900),
|
||||
'900-930 MHz (ISM Band - US)': (900, 930)
|
||||
}
|
||||
|
||||
for band_name, (min_freq, max_freq) in bands.items():
|
||||
matching = [s for s in signatures
|
||||
if min_freq <= (s['frequency']/1e6) <= max_freq]
|
||||
|
||||
print(f"\n{band_name}: {len(matching)} devices")
|
||||
|
||||
# Show first 10
|
||||
for sig in matching[:10]:
|
||||
print(f" - {sig['device_name']:40s} {sig['frequency']/1e6:7.2f} MHz {sig['protocol'] or 'RAW'}")
|
||||
|
||||
return signatures, parse_errors
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
flipper_dir = Path(__file__).parent.parent / 'signatures' / 'flipperzero-firmware'
|
||||
|
||||
if not flipper_dir.exists():
|
||||
print(f"❌ Flipper Zero directory not found: {flipper_dir}")
|
||||
print("Run: git clone https://github.com/flipperdevices/flipperzero-firmware.git")
|
||||
return 1
|
||||
|
||||
signatures, errors = analyze_flipper_database(flipper_dir)
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*80}")
|
||||
print("SUMMARY")
|
||||
print(f"{'='*80}")
|
||||
print(f"Total signatures: {len(signatures)}")
|
||||
print(f"Parse errors: {len(errors)}")
|
||||
print(f"\nThese signatures can now be imported into GigLez database")
|
||||
print(f"for device matching against wardriving captures.")
|
||||
print("="*80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Analyze T-Embed RF files and extract signatures
|
||||
|
||||
Shows what RF patterns we can extract from T-Embed captures
|
||||
without needing database connection
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
|
||||
|
||||
def analyze_rf_file(file_path: Path, parser: SubFileParser) -> Dict[str, Any]:
|
||||
"""Analyze a single RF file and extract features"""
|
||||
|
||||
try:
|
||||
metadata = parser.parse(str(file_path))
|
||||
|
||||
# Extract features
|
||||
features = {
|
||||
'filename': file_path.name,
|
||||
'parsed': True,
|
||||
'file_type': metadata.file_type,
|
||||
'frequency_hz': metadata.frequency,
|
||||
'frequency_mhz': metadata.frequency / 1e6 if metadata.frequency else 0,
|
||||
'protocol': metadata.protocol or 'RAW',
|
||||
'format': metadata.file_format,
|
||||
'modulation': metadata.modulation or 'Unknown'
|
||||
}
|
||||
|
||||
# RAW format specific features
|
||||
if metadata.raw_data and len(metadata.raw_data) > 0:
|
||||
abs_timings = [abs(t) for t in metadata.raw_data]
|
||||
|
||||
features.update({
|
||||
'raw_samples': len(metadata.raw_data),
|
||||
'timing_min': min(abs_timings),
|
||||
'timing_max': max(abs_timings),
|
||||
'timing_avg': sum(abs_timings) / len(abs_timings),
|
||||
'timing_range': max(abs_timings) - min(abs_timings),
|
||||
'raw_data_preview': metadata.raw_data[:20]
|
||||
})
|
||||
|
||||
# Calculate pattern characteristics
|
||||
features['pulse_count'] = len([t for t in metadata.raw_data if t > 0])
|
||||
features['gap_count'] = len([t for t in metadata.raw_data if t < 0])
|
||||
|
||||
# KEY format specific features
|
||||
if metadata.key_data:
|
||||
features.update({
|
||||
'key_data': metadata.key_data.hex(),
|
||||
'key_length': len(metadata.key_data),
|
||||
'bit_length': metadata.bit_length,
|
||||
'timing_element': metadata.timing_element
|
||||
})
|
||||
|
||||
# Empty file check
|
||||
if metadata.frequency == 0 or (metadata.file_format == 'RAW' and not metadata.raw_data):
|
||||
features['empty'] = True
|
||||
else:
|
||||
features['empty'] = False
|
||||
|
||||
return features
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
'filename': file_path.name,
|
||||
'parsed': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
|
||||
def generate_signature_from_features(features: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate a device signature from extracted features"""
|
||||
|
||||
if features.get('empty') or not features.get('parsed'):
|
||||
return None
|
||||
|
||||
signature = {
|
||||
'device_name': f"{features['filename'].replace('.sub', '')}_{features['frequency_mhz']:.0f}MHz",
|
||||
'frequency': features['frequency_hz'],
|
||||
'protocol': features['protocol'],
|
||||
'modulation': features['modulation']
|
||||
}
|
||||
|
||||
# Add timing signature for RAW
|
||||
if 'timing_min' in features:
|
||||
signature['timing_signature'] = {
|
||||
'min': features['timing_min'],
|
||||
'max': features['timing_max'],
|
||||
'avg': features['timing_avg'],
|
||||
'range': features['timing_range'],
|
||||
'samples': features['raw_samples']
|
||||
}
|
||||
|
||||
# Add pattern signature
|
||||
if 'raw_data_preview' in features:
|
||||
signature['pattern_preview'] = features['raw_data_preview']
|
||||
|
||||
# Guess device type from frequency
|
||||
freq_mhz = features['frequency_mhz']
|
||||
if 300 <= freq_mhz <= 350:
|
||||
signature['likely_type'] = 'Garage Door / Gate Opener'
|
||||
elif 400 <= freq_mhz <= 440:
|
||||
signature['likely_type'] = 'Remote Control / Key Fob'
|
||||
elif 860 <= freq_mhz <= 870:
|
||||
signature['likely_type'] = 'Sensor / RFID'
|
||||
elif 900 <= freq_mhz <= 930:
|
||||
signature['likely_type'] = 'ISM Device / Sensor / IoT'
|
||||
else:
|
||||
signature['likely_type'] = 'Unknown'
|
||||
|
||||
return signature
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*80)
|
||||
print("T-Embed RF File Analysis")
|
||||
print("="*80)
|
||||
print()
|
||||
|
||||
# Find T-Embed files
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
|
||||
if not tembed_dir.exists():
|
||||
print(f"❌ Directory not found: {tembed_dir}")
|
||||
return 1
|
||||
|
||||
sub_files = sorted(tembed_dir.glob('*.sub'))
|
||||
print(f"Found {len(sub_files)} .sub files\n")
|
||||
|
||||
parser = SubFileParser()
|
||||
|
||||
all_features = []
|
||||
signatures = []
|
||||
|
||||
# Analyze each file
|
||||
for sub_file in sub_files:
|
||||
print("-"*80)
|
||||
features = analyze_rf_file(sub_file, parser)
|
||||
all_features.append(features)
|
||||
|
||||
if not features.get('parsed'):
|
||||
print(f"❌ {features['filename']}: {features.get('error', 'Unknown error')}\n")
|
||||
continue
|
||||
|
||||
if features.get('empty'):
|
||||
print(f"⏭️ {features['filename']}: Empty capture (skipped)\n")
|
||||
continue
|
||||
|
||||
# Show analysis
|
||||
print(f"✅ {features['filename']}")
|
||||
print(f"\n Basic Info:")
|
||||
print(f" File Type: {features['file_type']}")
|
||||
print(f" Frequency: {features['frequency_mhz']:.2f} MHz ({features['frequency_hz']} Hz)")
|
||||
print(f" Protocol: {features['protocol']}")
|
||||
print(f" Format: {features['format']}")
|
||||
print(f" Modulation: {features['modulation']}")
|
||||
|
||||
if 'raw_samples' in features:
|
||||
print(f"\n RAW Signal Characteristics:")
|
||||
print(f" Samples: {features['raw_samples']}")
|
||||
print(f" Timing Range: {features['timing_min']}-{features['timing_max']} μs")
|
||||
print(f" Average Timing: {features['timing_avg']:.1f} μs")
|
||||
print(f" Pulse Count: {features['pulse_count']}")
|
||||
print(f" Gap Count: {features['gap_count']}")
|
||||
print(f" Preview: {features['raw_data_preview']}")
|
||||
|
||||
if 'key_data' in features:
|
||||
print(f"\n KEY Format Data:")
|
||||
print(f" Key: {features['key_data']}")
|
||||
print(f" Bit Length: {features['bit_length']}")
|
||||
print(f" Timing Element: {features['timing_element']}")
|
||||
|
||||
# Generate signature
|
||||
sig = generate_signature_from_features(features)
|
||||
if sig:
|
||||
signatures.append(sig)
|
||||
print(f"\n Device Signature:")
|
||||
print(f" Device Name: {sig['device_name']}")
|
||||
print(f" Likely Type: {sig['likely_type']}")
|
||||
|
||||
if 'timing_signature' in sig:
|
||||
ts = sig['timing_signature']
|
||||
print(f" Timing Signature: {ts['min']}-{ts['max']}μs (avg: {ts['avg']:.1f})")
|
||||
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print("="*80)
|
||||
print("ANALYSIS SUMMARY")
|
||||
print("="*80)
|
||||
|
||||
total = len(all_features)
|
||||
parsed = sum(1 for f in all_features if f.get('parsed'))
|
||||
empty = sum(1 for f in all_features if f.get('empty'))
|
||||
valid = sum(1 for f in all_features if f.get('parsed') and not f.get('empty'))
|
||||
|
||||
print(f"Total files: {total}")
|
||||
print(f"Successfully parsed: {parsed}")
|
||||
print(f"Empty captures: {empty}")
|
||||
print(f"Valid captures: {valid}")
|
||||
|
||||
print(f"\n Signatures Generated: {len(signatures)}")
|
||||
|
||||
if signatures:
|
||||
print("\nSignature Database Preview:")
|
||||
for i, sig in enumerate(signatures, 1):
|
||||
print(f"\n{i}. {sig['device_name']}")
|
||||
print(f" Frequency: {sig['frequency']/1e6:.2f} MHz")
|
||||
print(f" Type: {sig['likely_type']}")
|
||||
|
||||
if 'timing_signature' in sig:
|
||||
ts = sig['timing_signature']
|
||||
print(f" Timing: {ts['min']}-{ts['max']} μs ({ts['samples']} samples)")
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("✅ Analysis complete!")
|
||||
print("\nThese signatures can be imported into the database for device matching.")
|
||||
print("="*80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+442
@@ -0,0 +1,442 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deep RF Signal Analysis and Device Identification
|
||||
|
||||
Analyzes T-Embed captures and identifies likely devices based on:
|
||||
- Frequency band
|
||||
- Timing patterns
|
||||
- Pulse characteristics
|
||||
- Known device signatures in the 915 MHz ISM band
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Tuple
|
||||
import statistics
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
|
||||
|
||||
class RFSignalAnalyzer:
|
||||
"""Deep analysis of RF signals to identify device types"""
|
||||
|
||||
# Known 915 MHz ISM band devices and their characteristics
|
||||
KNOWN_915MHZ_DEVICES = {
|
||||
'wireless_sensor': {
|
||||
'name': 'Wireless Sensor (Temperature/Humidity)',
|
||||
'timing_range': (50, 1500),
|
||||
'avg_pulse_range': (200, 600),
|
||||
'pulse_count_range': (40, 100),
|
||||
'characteristics': ['Regular pulses', 'Short transmission bursts'],
|
||||
'manufacturers': ['Acurite', 'La Crosse', 'Oregon Scientific', 'Generic'],
|
||||
'confidence_multiplier': 0.9
|
||||
},
|
||||
'tpms': {
|
||||
'name': 'Tire Pressure Monitoring System (TPMS)',
|
||||
'timing_range': (30, 800),
|
||||
'avg_pulse_range': (100, 400),
|
||||
'pulse_count_range': (50, 150),
|
||||
'characteristics': ['Periodic transmission', 'Short data packets'],
|
||||
'manufacturers': ['Schrader', 'Continental', 'Sensata'],
|
||||
'confidence_multiplier': 0.85
|
||||
},
|
||||
'door_window_sensor': {
|
||||
'name': 'Door/Window Security Sensor',
|
||||
'timing_range': (100, 2000),
|
||||
'avg_pulse_range': (300, 800),
|
||||
'pulse_count_range': (20, 80),
|
||||
'characteristics': ['On-demand transmission', 'Low duty cycle'],
|
||||
'manufacturers': ['SimpliSafe', 'Ring', 'ADT', 'Generic'],
|
||||
'confidence_multiplier': 0.8
|
||||
},
|
||||
'utility_meter': {
|
||||
'name': 'Smart Utility Meter',
|
||||
'timing_range': (200, 3000),
|
||||
'avg_pulse_range': (400, 1200),
|
||||
'pulse_count_range': (100, 300),
|
||||
'characteristics': ['Regular interval transmission', 'Long packets'],
|
||||
'manufacturers': ['Itron', 'Landis+Gyr', 'Sensus'],
|
||||
'confidence_multiplier': 0.75
|
||||
},
|
||||
'motion_sensor': {
|
||||
'name': 'Motion Detector / PIR Sensor',
|
||||
'timing_range': (50, 1000),
|
||||
'avg_pulse_range': (150, 500),
|
||||
'pulse_count_range': (30, 90),
|
||||
'characteristics': ['Event-triggered', 'Quick bursts'],
|
||||
'manufacturers': ['Generic', 'Smart Home Brands'],
|
||||
'confidence_multiplier': 0.7
|
||||
},
|
||||
'remote_control': {
|
||||
'name': '915MHz Remote Control',
|
||||
'timing_range': (100, 2500),
|
||||
'avg_pulse_range': (250, 900),
|
||||
'pulse_count_range': (20, 70),
|
||||
'characteristics': ['Manual trigger', 'Short commands'],
|
||||
'manufacturers': ['Generic', 'Industrial'],
|
||||
'confidence_multiplier': 0.65
|
||||
},
|
||||
'iot_generic': {
|
||||
'name': 'Generic IoT Device',
|
||||
'timing_range': (10, 5000),
|
||||
'avg_pulse_range': (50, 2000),
|
||||
'pulse_count_range': (10, 500),
|
||||
'characteristics': ['Variable patterns'],
|
||||
'manufacturers': ['Various'],
|
||||
'confidence_multiplier': 0.5
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.parser = SubFileParser()
|
||||
|
||||
def analyze_file(self, file_path: Path) -> Dict[str, Any]:
|
||||
"""Perform deep analysis on a .sub file"""
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"ANALYZING: {file_path.name}")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Parse file
|
||||
try:
|
||||
metadata = self.parser.parse(str(file_path))
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
|
||||
# Check if valid
|
||||
if metadata.frequency == 0 or (metadata.file_format == 'RAW' and not metadata.raw_data):
|
||||
return {'skipped': True, 'reason': 'Empty capture'}
|
||||
|
||||
# Basic info
|
||||
print("BASIC SIGNAL INFORMATION")
|
||||
print("-" * 80)
|
||||
print(f"File Type: {metadata.file_type}")
|
||||
print(f"Frequency: {metadata.frequency/1e6:.3f} MHz ({metadata.frequency} Hz)")
|
||||
print(f"Protocol: {metadata.protocol or 'RAW (undecoded)'}")
|
||||
print(f"Format: {metadata.file_format}")
|
||||
print(f"Modulation: {metadata.modulation or 'Unknown'}")
|
||||
|
||||
# Analyze RAW data
|
||||
if not metadata.raw_data:
|
||||
print("\nNo RAW data to analyze")
|
||||
return {'error': 'No RAW data'}
|
||||
|
||||
analysis = self._analyze_timing(metadata.raw_data)
|
||||
|
||||
print(f"\nRAW TIMING ANALYSIS")
|
||||
print("-" * 80)
|
||||
print(f"Total Samples: {analysis['total_samples']}")
|
||||
print(f"Pulse Count: {analysis['pulse_count']} (positive values)")
|
||||
print(f"Gap Count: {analysis['gap_count']} (negative values)")
|
||||
print(f"\nTiming Statistics (microseconds):")
|
||||
print(f" Min: {analysis['timing_min']} μs")
|
||||
print(f" Max: {analysis['timing_max']} μs")
|
||||
print(f" Average: {analysis['timing_avg']:.2f} μs")
|
||||
print(f" Median: {analysis['timing_median']:.2f} μs")
|
||||
print(f" Std Dev: {analysis['timing_stddev']:.2f} μs")
|
||||
print(f"\nPulse Width Analysis:")
|
||||
print(f" Avg Pulse: {analysis['avg_pulse_width']:.2f} μs")
|
||||
print(f" Avg Gap: {analysis['avg_gap_width']:.2f} μs")
|
||||
print(f" Pulse/Gap: {analysis['pulse_gap_ratio']:.2f}")
|
||||
|
||||
# Pattern analysis
|
||||
pattern_analysis = self._analyze_pattern(metadata.raw_data)
|
||||
|
||||
print(f"\nPATTERN CHARACTERISTICS")
|
||||
print("-" * 80)
|
||||
print(f"Repeating Patterns: {pattern_analysis['has_repetition']}")
|
||||
print(f"Pattern Regularity: {pattern_analysis['regularity']}")
|
||||
print(f"Transmission Type: {pattern_analysis['transmission_type']}")
|
||||
|
||||
# Device identification
|
||||
print(f"\n{'='*80}")
|
||||
print("DEVICE IDENTIFICATION")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Match against known devices
|
||||
matches = self._identify_device(metadata.frequency, analysis, pattern_analysis)
|
||||
|
||||
if matches:
|
||||
print(f"Found {len(matches)} potential match(es):\n")
|
||||
|
||||
for i, match in enumerate(matches, 1):
|
||||
print(f"{i}. {match['name']}")
|
||||
print(f" Confidence: {match['confidence']:.1%}")
|
||||
print(f" Match Score: {match['score']:.2f}/1.0")
|
||||
print(f" Manufacturers: {', '.join(match['manufacturers'])}")
|
||||
print(f" Characteristics: {', '.join(match['characteristics'])}")
|
||||
print(f"\n Match Details:")
|
||||
|
||||
for detail_key, detail_val in match['match_details'].items():
|
||||
print(f" {detail_key}: {detail_val}")
|
||||
|
||||
print()
|
||||
|
||||
# Best match
|
||||
best = matches[0]
|
||||
print(f"{'='*80}")
|
||||
print(f"MOST LIKELY DEVICE: {best['name']}")
|
||||
print(f"Confidence: {best['confidence']:.1%}")
|
||||
print(f"{'='*80}")
|
||||
|
||||
else:
|
||||
print("❌ No matches found in known device database")
|
||||
print("\nThis could be:")
|
||||
print(" - A custom/proprietary device")
|
||||
print(" - A new/unknown protocol")
|
||||
print(" - Interference or noise")
|
||||
|
||||
return {
|
||||
'file': file_path.name,
|
||||
'frequency': metadata.frequency,
|
||||
'analysis': analysis,
|
||||
'pattern': pattern_analysis,
|
||||
'matches': matches
|
||||
}
|
||||
|
||||
def _analyze_timing(self, raw_data: List[int]) -> Dict[str, Any]:
|
||||
"""Analyze timing characteristics"""
|
||||
|
||||
abs_timings = [abs(t) for t in raw_data]
|
||||
pulses = [t for t in raw_data if t > 0]
|
||||
gaps = [abs(t) for t in raw_data if t < 0]
|
||||
|
||||
analysis = {
|
||||
'total_samples': len(raw_data),
|
||||
'pulse_count': len(pulses),
|
||||
'gap_count': len(gaps),
|
||||
'timing_min': min(abs_timings),
|
||||
'timing_max': max(abs_timings),
|
||||
'timing_avg': statistics.mean(abs_timings),
|
||||
'timing_median': statistics.median(abs_timings),
|
||||
'timing_stddev': statistics.stdev(abs_timings) if len(abs_timings) > 1 else 0,
|
||||
}
|
||||
|
||||
if pulses:
|
||||
analysis['avg_pulse_width'] = statistics.mean(pulses)
|
||||
else:
|
||||
analysis['avg_pulse_width'] = 0
|
||||
|
||||
if gaps:
|
||||
analysis['avg_gap_width'] = statistics.mean(gaps)
|
||||
else:
|
||||
analysis['avg_gap_width'] = 0
|
||||
|
||||
if analysis['avg_gap_width'] > 0:
|
||||
analysis['pulse_gap_ratio'] = analysis['avg_pulse_width'] / analysis['avg_gap_width']
|
||||
else:
|
||||
analysis['pulse_gap_ratio'] = 0
|
||||
|
||||
return analysis
|
||||
|
||||
def _analyze_pattern(self, raw_data: List[int]) -> Dict[str, Any]:
|
||||
"""Analyze signal patterns"""
|
||||
|
||||
# Check for repetition
|
||||
has_repetition = self._check_repetition(raw_data)
|
||||
|
||||
# Calculate regularity (coefficient of variation)
|
||||
abs_timings = [abs(t) for t in raw_data]
|
||||
avg = statistics.mean(abs_timings)
|
||||
stddev = statistics.stdev(abs_timings) if len(abs_timings) > 1 else 0
|
||||
cv = (stddev / avg) if avg > 0 else 0
|
||||
|
||||
if cv < 0.5:
|
||||
regularity = "High (uniform timing)"
|
||||
elif cv < 1.5:
|
||||
regularity = "Moderate (some variation)"
|
||||
else:
|
||||
regularity = "Low (highly variable)"
|
||||
|
||||
# Determine transmission type
|
||||
if cv < 0.7 and has_repetition:
|
||||
transmission_type = "Periodic (sensor/beacon)"
|
||||
elif cv > 2.0:
|
||||
transmission_type = "Bursty (on-demand)"
|
||||
else:
|
||||
transmission_type = "Mixed (varies)"
|
||||
|
||||
return {
|
||||
'has_repetition': has_repetition,
|
||||
'regularity': regularity,
|
||||
'coefficient_variation': cv,
|
||||
'transmission_type': transmission_type
|
||||
}
|
||||
|
||||
def _check_repetition(self, raw_data: List[int], window_size: int = 10) -> bool:
|
||||
"""Check if pattern has repetition"""
|
||||
|
||||
if len(raw_data) < window_size * 2:
|
||||
return False
|
||||
|
||||
# Simple check: see if first window repeats
|
||||
window1 = raw_data[:window_size]
|
||||
|
||||
for i in range(window_size, len(raw_data) - window_size):
|
||||
window2 = raw_data[i:i+window_size]
|
||||
|
||||
# Check similarity
|
||||
matches = sum(1 for j in range(window_size)
|
||||
if abs(window1[j] - window2[j]) < abs(window1[j]) * 0.2)
|
||||
|
||||
if matches >= window_size * 0.7: # 70% similarity
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _identify_device(self, frequency: int, timing_analysis: Dict,
|
||||
pattern_analysis: Dict) -> List[Dict[str, Any]]:
|
||||
"""Identify device based on RF characteristics"""
|
||||
|
||||
freq_mhz = frequency / 1e6
|
||||
|
||||
# Only process 915 MHz ISM band
|
||||
if not (900 <= freq_mhz <= 930):
|
||||
return []
|
||||
|
||||
matches = []
|
||||
|
||||
for device_key, device_info in self.KNOWN_915MHZ_DEVICES.items():
|
||||
score = 0.0
|
||||
match_details = {}
|
||||
|
||||
# Check timing range
|
||||
timing_match = self._check_range_match(
|
||||
timing_analysis['timing_avg'],
|
||||
device_info['timing_range']
|
||||
)
|
||||
score += timing_match * 0.3
|
||||
match_details['Timing Match'] = f"{timing_match:.1%}"
|
||||
|
||||
# Check average pulse
|
||||
pulse_match = self._check_range_match(
|
||||
timing_analysis['avg_pulse_width'],
|
||||
device_info['avg_pulse_range']
|
||||
)
|
||||
score += pulse_match * 0.3
|
||||
match_details['Pulse Match'] = f"{pulse_match:.1%}"
|
||||
|
||||
# Check pulse count
|
||||
pulse_count_match = self._check_range_match(
|
||||
timing_analysis['pulse_count'],
|
||||
device_info['pulse_count_range']
|
||||
)
|
||||
score += pulse_count_match * 0.2
|
||||
match_details['Count Match'] = f"{pulse_count_match:.1%}"
|
||||
|
||||
# Pattern characteristics bonus
|
||||
if 'Periodic' in pattern_analysis['transmission_type'] and 'sensor' in device_key:
|
||||
score += 0.1
|
||||
match_details['Pattern Bonus'] = 'Periodic transmission (sensor-like)'
|
||||
|
||||
if 'Bursty' in pattern_analysis['transmission_type'] and 'remote' in device_key:
|
||||
score += 0.1
|
||||
match_details['Pattern Bonus'] = 'Bursty transmission (control-like)'
|
||||
|
||||
# Only include if reasonable match
|
||||
if score > 0.3:
|
||||
confidence = score * device_info['confidence_multiplier']
|
||||
|
||||
matches.append({
|
||||
'device_key': device_key,
|
||||
'name': device_info['name'],
|
||||
'confidence': confidence,
|
||||
'score': score,
|
||||
'manufacturers': device_info['manufacturers'],
|
||||
'characteristics': device_info['characteristics'],
|
||||
'match_details': match_details
|
||||
})
|
||||
|
||||
# Sort by confidence
|
||||
matches.sort(key=lambda x: x['confidence'], reverse=True)
|
||||
|
||||
return matches
|
||||
|
||||
def _check_range_match(self, value: float, range_tuple: Tuple[float, float]) -> float:
|
||||
"""
|
||||
Check how well a value fits within a range
|
||||
|
||||
Returns: 0.0-1.0 score
|
||||
"""
|
||||
min_val, max_val = range_tuple
|
||||
|
||||
if min_val <= value <= max_val:
|
||||
# Value is within range
|
||||
center = (min_val + max_val) / 2
|
||||
distance = abs(value - center)
|
||||
range_size = (max_val - min_val) / 2
|
||||
|
||||
# Score decreases as we move from center
|
||||
score = 1.0 - (distance / range_size) if range_size > 0 else 1.0
|
||||
return max(0.5, score) # At least 0.5 if in range
|
||||
|
||||
elif value < min_val:
|
||||
# Below range
|
||||
distance = min_val - value
|
||||
return max(0.0, 1.0 - (distance / min_val))
|
||||
|
||||
else:
|
||||
# Above range
|
||||
distance = value - max_val
|
||||
return max(0.0, 1.0 - (distance / max_val))
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*80)
|
||||
print("T-EMBED RF DEVICE IDENTIFICATION")
|
||||
print("Deep Signal Analysis & Device Detection")
|
||||
print("="*80)
|
||||
|
||||
# Find T-Embed files
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
|
||||
if not tembed_dir.exists():
|
||||
print(f"❌ Directory not found: {tembed_dir}")
|
||||
return 1
|
||||
|
||||
sub_files = sorted(tembed_dir.glob('*.sub'))
|
||||
print(f"\nFound {len(sub_files)} .sub files to analyze\n")
|
||||
|
||||
analyzer = RFSignalAnalyzer()
|
||||
results = []
|
||||
|
||||
# Analyze each file
|
||||
for sub_file in sub_files:
|
||||
result = analyzer.analyze_file(sub_file)
|
||||
if 'error' not in result and 'skipped' not in result:
|
||||
results.append(result)
|
||||
|
||||
# Final summary
|
||||
print(f"\n{'='*80}")
|
||||
print("SUMMARY: DEVICES DETECTED")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
if results:
|
||||
for i, result in enumerate(results, 1):
|
||||
print(f"{i}. {result['file']}")
|
||||
print(f" Frequency: {result['frequency']/1e6:.2f} MHz")
|
||||
|
||||
if result['matches']:
|
||||
best_match = result['matches'][0]
|
||||
print(f" Identified: {best_match['name']}")
|
||||
print(f" Confidence: {best_match['confidence']:.1%}")
|
||||
print(f" Likely Manufacturer: {best_match['manufacturers'][0]}")
|
||||
else:
|
||||
print(f" Identified: Unknown device")
|
||||
|
||||
print()
|
||||
else:
|
||||
print("No valid devices detected (all files were empty or parse errors)\n")
|
||||
|
||||
print("="*80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import Flipper Zero signatures into SQLite database
|
||||
|
||||
Uses SQLite for immediate testing without PostgreSQL setup
|
||||
"""
|
||||
|
||||
import sys
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
|
||||
|
||||
def create_sqlite_schema(conn):
|
||||
"""Create SQLite schema"""
|
||||
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Devices table
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS devices (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
device_name TEXT,
|
||||
manufacturer TEXT,
|
||||
model TEXT,
|
||||
device_type TEXT,
|
||||
typical_frequency INTEGER,
|
||||
protocol TEXT,
|
||||
description TEXT,
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
is_verified BOOLEAN DEFAULT 0,
|
||||
source TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
# Signatures table
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS signatures (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
device_id INTEGER REFERENCES devices(id),
|
||||
protocol TEXT,
|
||||
frequency INTEGER,
|
||||
modulation TEXT,
|
||||
bit_pattern BLOB,
|
||||
bit_mask BLOB,
|
||||
timing_min INTEGER,
|
||||
timing_max INTEGER,
|
||||
raw_pattern TEXT,
|
||||
confidence_threshold REAL DEFAULT 0.7,
|
||||
source TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# Indexes
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_sig_freq ON signatures(frequency)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_sig_device ON signatures(device_id)')
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
||||
def import_flipper_signature(conn, sub_file: Path, parser: SubFileParser):
|
||||
"""Import a single Flipper Zero .sub file"""
|
||||
|
||||
try:
|
||||
metadata = parser.parse(str(sub_file))
|
||||
|
||||
# Skip if frequency is 0
|
||||
if metadata.frequency == 0:
|
||||
return None
|
||||
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Determine device type from frequency
|
||||
freq_mhz = metadata.frequency / 1e6
|
||||
|
||||
if 300 <= freq_mhz <= 350:
|
||||
device_type = 'garage_door'
|
||||
elif 400 <= freq_mhz <= 450:
|
||||
device_type = 'remote_control'
|
||||
elif 800 <= freq_mhz <= 900:
|
||||
device_type = 'sensor'
|
||||
else:
|
||||
device_type = 'unknown'
|
||||
|
||||
# Create device record
|
||||
cursor.execute('''
|
||||
INSERT INTO devices (device_name, manufacturer, model, device_type,
|
||||
typical_frequency, protocol, description, source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
sub_file.stem,
|
||||
'Unknown',
|
||||
sub_file.stem,
|
||||
device_type,
|
||||
metadata.frequency,
|
||||
metadata.protocol or 'RAW',
|
||||
f'Imported from Flipper Zero: {sub_file.name}',
|
||||
'flipper_zero'
|
||||
))
|
||||
|
||||
device_id = cursor.lastrowid
|
||||
|
||||
# Extract timing info
|
||||
timing_min, timing_max = None, None
|
||||
raw_pattern = None
|
||||
|
||||
if metadata.raw_data and len(metadata.raw_data) > 0:
|
||||
abs_timings = [abs(t) for t in metadata.raw_data]
|
||||
timing_min = min(abs_timings)
|
||||
timing_max = max(abs_timings)
|
||||
raw_pattern = ','.join(map(str, metadata.raw_data[:100])) # First 100 samples
|
||||
|
||||
# Create signature record
|
||||
cursor.execute('''
|
||||
INSERT INTO signatures (device_id, protocol, frequency, modulation,
|
||||
timing_min, timing_max, raw_pattern, source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
device_id,
|
||||
metadata.protocol or 'RAW',
|
||||
metadata.frequency,
|
||||
metadata.modulation,
|
||||
timing_min,
|
||||
timing_max,
|
||||
raw_pattern,
|
||||
'flipper_zero'
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
|
||||
return {
|
||||
'device_id': device_id,
|
||||
'device_name': sub_file.stem,
|
||||
'frequency': metadata.frequency,
|
||||
'protocol': metadata.protocol
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*80)
|
||||
print("FLIPPER ZERO → SQLite IMPORT")
|
||||
print("="*80)
|
||||
print()
|
||||
|
||||
# Create/connect to SQLite database
|
||||
db_path = Path(__file__).parent.parent / 'giglez.db'
|
||||
|
||||
print(f"Database: {db_path}")
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
print("✅ Connected to SQLite database\n")
|
||||
|
||||
# Create schema
|
||||
print("Creating schema...")
|
||||
create_sqlite_schema(conn)
|
||||
print("✅ Schema ready\n")
|
||||
|
||||
# Find Flipper signatures
|
||||
flipper_dir = Path(__file__).parent.parent / 'signatures' / 'flipperzero-firmware'
|
||||
|
||||
if not flipper_dir.exists():
|
||||
print(f"❌ Flipper directory not found: {flipper_dir}")
|
||||
return 1
|
||||
|
||||
sub_files = list(flipper_dir.glob('**/*.sub'))
|
||||
print(f"Found {len(sub_files)} Flipper Zero .sub files\n")
|
||||
|
||||
# Import all signatures
|
||||
parser = SubFileParser()
|
||||
imported = []
|
||||
skipped = 0
|
||||
|
||||
print("Importing signatures...")
|
||||
for i, sub_file in enumerate(sub_files, 1):
|
||||
if i % 10 == 0:
|
||||
print(f" Processed {i}/{len(sub_files)}...")
|
||||
|
||||
result = import_flipper_signature(conn, sub_file, parser)
|
||||
|
||||
if result:
|
||||
imported.append(result)
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
print(f"✅ Import complete\n")
|
||||
|
||||
# Summary
|
||||
print("="*80)
|
||||
print("IMPORT SUMMARY")
|
||||
print("="*80)
|
||||
print(f"Total files: {len(sub_files)}")
|
||||
print(f"Imported: {len(imported)}")
|
||||
print(f"Skipped: {skipped}")
|
||||
|
||||
# Query database
|
||||
cursor = conn.cursor()
|
||||
|
||||
print(f"\nDATABASE CONTENTS")
|
||||
print("-"*80)
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM devices")
|
||||
device_count = cursor.fetchone()[0]
|
||||
print(f"Devices: {device_count}")
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM signatures")
|
||||
sig_count = cursor.fetchone()[0]
|
||||
print(f"Signatures: {sig_count}")
|
||||
|
||||
# Frequency distribution
|
||||
print(f"\nFREQUENCY DISTRIBUTION")
|
||||
print("-"*80)
|
||||
|
||||
cursor.execute('''
|
||||
SELECT frequency, COUNT(*) as count
|
||||
FROM signatures
|
||||
GROUP BY frequency
|
||||
ORDER BY count DESC
|
||||
''')
|
||||
|
||||
for freq, count in cursor.fetchall():
|
||||
print(f"{freq/1e6:8.2f} MHz: {count:3d} devices")
|
||||
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("✅ Database ready at:", db_path)
|
||||
print("="*80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+329
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import T-Embed RF captures as signature database entries
|
||||
|
||||
This script:
|
||||
1. Scans T-Embed .sub files
|
||||
2. Extracts RF signal patterns
|
||||
3. Creates device and signature records
|
||||
4. Populates database for matching
|
||||
|
||||
Based on real wardriving captures from T-Embed device
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
from src.database.models import Device, Signature, FlipperSignature, Base
|
||||
from src.database.connection import get_engine, get_session
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
||||
class TembedSignatureImporter:
|
||||
"""Import T-Embed RF captures as device signatures"""
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self.session = session
|
||||
self.parser = SubFileParser()
|
||||
self.stats = {
|
||||
'files_found': 0,
|
||||
'files_parsed': 0,
|
||||
'files_skipped': 0,
|
||||
'devices_created': 0,
|
||||
'signatures_created': 0,
|
||||
'errors': []
|
||||
}
|
||||
|
||||
def import_directory(self, directory: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
Import all .sub files from directory
|
||||
|
||||
Args:
|
||||
directory: Path to directory containing .sub files
|
||||
|
||||
Returns:
|
||||
Statistics dictionary
|
||||
"""
|
||||
print(f"Scanning {directory} for .sub files...")
|
||||
|
||||
# Find all .sub files
|
||||
sub_files = list(directory.glob('*.sub'))
|
||||
self.stats['files_found'] = len(sub_files)
|
||||
|
||||
print(f"Found {len(sub_files)} .sub files\n")
|
||||
|
||||
for sub_file in sorted(sub_files):
|
||||
print(f"Processing: {sub_file.name}")
|
||||
try:
|
||||
self._import_file(sub_file)
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing {sub_file.name}: {e}"
|
||||
print(f" ❌ {error_msg}")
|
||||
self.stats['errors'].append(error_msg)
|
||||
|
||||
# Commit all changes
|
||||
try:
|
||||
self.session.commit()
|
||||
print("\n✅ Database changes committed")
|
||||
except Exception as e:
|
||||
self.session.rollback()
|
||||
print(f"\n❌ Failed to commit: {e}")
|
||||
self.stats['errors'].append(f"Commit failed: {e}")
|
||||
|
||||
return self.stats
|
||||
|
||||
def _import_file(self, file_path: Path):
|
||||
"""Import a single .sub file"""
|
||||
|
||||
# Parse file
|
||||
try:
|
||||
metadata = self.parser.parse(str(file_path))
|
||||
except Exception as e:
|
||||
self.stats['files_skipped'] += 1
|
||||
raise ValueError(f"Parse failed: {e}")
|
||||
|
||||
# Skip if empty (frequency = 0, no data)
|
||||
if metadata.frequency == 0 or (metadata.file_format == 'RAW' and not metadata.raw_data):
|
||||
print(f" ⏭️ Skipped: Empty capture")
|
||||
self.stats['files_skipped'] += 1
|
||||
return
|
||||
|
||||
self.stats['files_parsed'] += 1
|
||||
|
||||
# Load GPS data if available
|
||||
gps_data = self._load_gps_data(file_path)
|
||||
|
||||
# Create device record
|
||||
device = self._create_device(metadata, file_path, gps_data)
|
||||
|
||||
# Create signature record
|
||||
signature = self._create_signature(metadata, device)
|
||||
|
||||
# Create Flipper signature record (for compatibility)
|
||||
flipper_sig = self._create_flipper_signature(metadata, device, file_path)
|
||||
|
||||
print(f" ✅ Device: {device.device_name}")
|
||||
print(f" Frequency: {metadata.frequency/1e6:.2f} MHz")
|
||||
if metadata.raw_data:
|
||||
print(f" RAW samples: {len(metadata.raw_data)}")
|
||||
if gps_data:
|
||||
print(f" GPS: {gps_data['data']['latitude']:.4f}, {gps_data['data']['longitude']:.4f}")
|
||||
|
||||
def _load_gps_data(self, sub_file: Path) -> Optional[Dict]:
|
||||
"""Load GPS coordinates for a .sub file if available"""
|
||||
|
||||
# Look for matching GPS JSON files in same directory
|
||||
# Pattern: gps_coordinates_YYYYMMDD_HHMMSS.json
|
||||
gps_files = sorted(sub_file.parent.glob('gps_coordinates_*.json'))
|
||||
|
||||
if not gps_files:
|
||||
return None
|
||||
|
||||
# Use the most recent GPS file (simple heuristic)
|
||||
gps_file = gps_files[-1]
|
||||
|
||||
try:
|
||||
with open(gps_file, 'r') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Could not load GPS data: {e}")
|
||||
return None
|
||||
|
||||
def _create_device(self, metadata, file_path: Path, gps_data: Optional[Dict]) -> Device:
|
||||
"""Create a Device record from metadata"""
|
||||
|
||||
# Generate device name from file and frequency
|
||||
device_name = self._generate_device_name(file_path, metadata)
|
||||
|
||||
# Determine device type from frequency
|
||||
device_type = self._guess_device_type(metadata.frequency)
|
||||
|
||||
# Check if device already exists
|
||||
existing = self.session.query(Device).filter(
|
||||
Device.device_name == device_name
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
print(f" ℹ️ Device already exists: {device_name}")
|
||||
return existing
|
||||
|
||||
# Create new device
|
||||
device = Device(
|
||||
device_name=device_name,
|
||||
manufacturer='Unknown',
|
||||
model='T-Embed Capture',
|
||||
device_type=device_type,
|
||||
typical_frequency=metadata.frequency,
|
||||
protocol=metadata.protocol if metadata.protocol else 'RAW',
|
||||
description=f"Captured from T-Embed RF device at {file_path.name}",
|
||||
first_seen=datetime.utcnow(),
|
||||
is_verified=False
|
||||
)
|
||||
|
||||
self.session.add(device)
|
||||
self.session.flush() # Get device.id
|
||||
|
||||
self.stats['devices_created'] += 1
|
||||
|
||||
return device
|
||||
|
||||
def _create_signature(self, metadata, device: Device) -> Signature:
|
||||
"""Create a Signature record for pattern matching"""
|
||||
|
||||
# Check if signature already exists
|
||||
existing = self.session.query(Signature).filter(
|
||||
Signature.device_id == device.id,
|
||||
Signature.frequency == metadata.frequency
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
# Extract timing patterns for RAW format
|
||||
timing_min, timing_max = None, None
|
||||
if metadata.raw_data and len(metadata.raw_data) > 0:
|
||||
# Use absolute values for timing
|
||||
abs_timings = [abs(t) for t in metadata.raw_data]
|
||||
timing_min = min(abs_timings)
|
||||
timing_max = max(abs_timings)
|
||||
|
||||
signature = Signature(
|
||||
device_id=device.id,
|
||||
protocol=metadata.protocol if metadata.protocol else 'RAW',
|
||||
frequency=metadata.frequency,
|
||||
modulation=metadata.modulation,
|
||||
bit_pattern=None, # Not available for RAW
|
||||
bit_mask=None,
|
||||
timing_min=timing_min,
|
||||
timing_max=timing_max,
|
||||
raw_pattern=','.join(map(str, metadata.raw_data)) if metadata.raw_data else None,
|
||||
confidence_threshold=0.7, # Default threshold
|
||||
source='tembed_wardriving',
|
||||
created_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
self.session.add(signature)
|
||||
self.stats['signatures_created'] += 1
|
||||
|
||||
return signature
|
||||
|
||||
def _create_flipper_signature(self, metadata, device: Device, file_path: Path) -> FlipperSignature:
|
||||
"""Create FlipperSignature record for compatibility"""
|
||||
|
||||
# Check if exists
|
||||
existing = self.session.query(FlipperSignature).filter(
|
||||
FlipperSignature.device_id == device.id
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
flipper_sig = FlipperSignature(
|
||||
device_id=device.id,
|
||||
frequency=metadata.frequency,
|
||||
preset=metadata.preset if metadata.preset else '0',
|
||||
protocol=metadata.protocol if metadata.protocol else 'RAW',
|
||||
bit=metadata.bit_length,
|
||||
key=metadata.key_data,
|
||||
te=metadata.timing_element,
|
||||
raw_data=','.join(map(str, metadata.raw_data)) if metadata.raw_data else None,
|
||||
source_file=file_path.name,
|
||||
imported_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
self.session.add(flipper_sig)
|
||||
|
||||
return flipper_sig
|
||||
|
||||
def _generate_device_name(self, file_path: Path, metadata) -> str:
|
||||
"""Generate a unique device name"""
|
||||
freq_mhz = metadata.frequency / 1e6
|
||||
return f"{file_path.stem}_{freq_mhz:.0f}MHz"
|
||||
|
||||
def _guess_device_type(self, frequency: int) -> str:
|
||||
"""Guess device type from frequency"""
|
||||
freq_mhz = frequency / 1e6
|
||||
|
||||
if 300 <= freq_mhz <= 350:
|
||||
return 'garage_door'
|
||||
elif 400 <= freq_mhz <= 440:
|
||||
return 'remote_control'
|
||||
elif 860 <= freq_mhz <= 870:
|
||||
return 'sensor'
|
||||
elif 900 <= freq_mhz <= 930:
|
||||
return 'ism_device' # 915 MHz ISM band
|
||||
else:
|
||||
return 'unknown'
|
||||
|
||||
def print_summary(self):
|
||||
"""Print import summary"""
|
||||
print("\n" + "="*60)
|
||||
print("IMPORT SUMMARY")
|
||||
print("="*60)
|
||||
print(f"Files found: {self.stats['files_found']}")
|
||||
print(f"Files parsed: {self.stats['files_parsed']}")
|
||||
print(f"Files skipped: {self.stats['files_skipped']}")
|
||||
print(f"Devices created: {self.stats['devices_created']}")
|
||||
print(f"Signatures created: {self.stats['signatures_created']}")
|
||||
|
||||
if self.stats['errors']:
|
||||
print(f"\nErrors: {len(self.stats['errors'])}")
|
||||
for error in self.stats['errors']:
|
||||
print(f" - {error}")
|
||||
else:
|
||||
print("\n✅ No errors")
|
||||
|
||||
print("="*60)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*60)
|
||||
print("T-Embed RF Signature Importer")
|
||||
print("="*60)
|
||||
print()
|
||||
|
||||
# Get database session
|
||||
try:
|
||||
engine = get_engine()
|
||||
session = get_session()
|
||||
print("✅ Database connected")
|
||||
except Exception as e:
|
||||
print(f"❌ Database connection failed: {e}")
|
||||
print("\nMake sure PostgreSQL is running and configured correctly")
|
||||
return 1
|
||||
|
||||
# Create tables if needed
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
print("✅ Database tables ready\n")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not create tables: {e}\n")
|
||||
|
||||
# Find T-Embed directory
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
|
||||
if not tembed_dir.exists():
|
||||
print(f"❌ Directory not found: {tembed_dir}")
|
||||
return 1
|
||||
|
||||
# Import signatures
|
||||
importer = TembedSignatureImporter(session)
|
||||
stats = importer.import_directory(tembed_dir)
|
||||
importer.print_summary()
|
||||
|
||||
session.close()
|
||||
|
||||
return 0 if not stats['errors'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Match T-Embed capture against populated database
|
||||
|
||||
Final demonstration of device identification with real signature database
|
||||
"""
|
||||
|
||||
import sys
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
|
||||
|
||||
def match_by_frequency(conn, target_freq: int, tolerance_hz: int = 10000):
|
||||
"""Match by frequency with tolerance"""
|
||||
|
||||
cursor = conn.cursor()
|
||||
|
||||
freq_min = target_freq - tolerance_hz
|
||||
freq_max = target_freq + tolerance_hz
|
||||
|
||||
cursor.execute('''
|
||||
SELECT d.device_name, d.protocol, s.frequency, s.timing_min, s.timing_max
|
||||
FROM devices d
|
||||
JOIN signatures s ON s.device_id = d.id
|
||||
WHERE s.frequency BETWEEN ? AND ?
|
||||
ORDER BY ABS(s.frequency - ?) ASC
|
||||
LIMIT 10
|
||||
''', (freq_min, freq_max, target_freq))
|
||||
|
||||
matches = []
|
||||
for row in cursor.fetchall():
|
||||
device_name, protocol, freq, timing_min, timing_max = row
|
||||
|
||||
freq_diff = abs(freq - target_freq)
|
||||
confidence = 1.0 - (freq_diff / tolerance_hz)
|
||||
confidence = max(0.5, confidence)
|
||||
|
||||
matches.append({
|
||||
'device_name': device_name,
|
||||
'protocol': protocol,
|
||||
'frequency': freq,
|
||||
'timing_min': timing_min,
|
||||
'timing_max': timing_max,
|
||||
'freq_diff_hz': freq_diff,
|
||||
'confidence': confidence
|
||||
})
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*80)
|
||||
print("DEVICE MATCHING: T-Embed vs Database")
|
||||
print("="*80)
|
||||
print()
|
||||
|
||||
# Connect to database
|
||||
db_path = Path(__file__).parent.parent / 'giglez.db'
|
||||
|
||||
if not db_path.exists():
|
||||
print(f"❌ Database not found: {db_path}")
|
||||
print("Run: python3 scripts/import_flipper_sqlite.py")
|
||||
return 1
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
print(f"✅ Connected to database: {db_path}\n")
|
||||
|
||||
# Check database contents
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM devices")
|
||||
device_count = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM signatures")
|
||||
sig_count = cursor.fetchone()[0]
|
||||
|
||||
print(f"Database contents:")
|
||||
print(f" Devices: {device_count}")
|
||||
print(f" Signatures: {sig_count}\n")
|
||||
|
||||
# Parse T-Embed capture
|
||||
tembed_file = Path(__file__).parent.parent / 'signatures' / 't-embed-rf' / 'raw_7.sub'
|
||||
|
||||
print(f"Analyzing: {tembed_file.name}")
|
||||
print("-"*80)
|
||||
|
||||
parser = SubFileParser()
|
||||
metadata = parser.parse(str(tembed_file))
|
||||
|
||||
print(f"Frequency: {metadata.frequency/1e6:.2f} MHz")
|
||||
print(f"Protocol: {metadata.protocol or 'RAW (undecoded)'}")
|
||||
print(f"Format: {metadata.file_format}")
|
||||
|
||||
if metadata.raw_data:
|
||||
abs_timings = [abs(t) for t in metadata.raw_data]
|
||||
print(f"RAW Samples: {len(metadata.raw_data)}")
|
||||
print(f"Timing Range: {min(abs_timings)}-{max(abs_timings)} μs")
|
||||
|
||||
# Match against database
|
||||
print(f"\n{'='*80}")
|
||||
print("MATCHING AGAINST DATABASE")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
matches = match_by_frequency(conn, metadata.frequency, tolerance_hz=500000000) # 500 MHz tolerance
|
||||
|
||||
if matches:
|
||||
print(f"Found {len(matches)} potential matches:\n")
|
||||
|
||||
for i, match in enumerate(matches, 1):
|
||||
print(f"{i}. {match['device_name']}")
|
||||
print(f" Frequency: {match['frequency']/1e6:.2f} MHz (diff: {match['freq_diff_hz']/1e6:.1f} MHz)")
|
||||
print(f" Protocol: {match['protocol']}")
|
||||
|
||||
if match['timing_min'] and match['timing_max']:
|
||||
print(f" Timing: {match['timing_min']}-{match['timing_max']} μs")
|
||||
|
||||
print(f" Confidence: {match['confidence']:.1%}")
|
||||
print()
|
||||
|
||||
# Best match
|
||||
best = matches[0]
|
||||
print(f"{'='*80}")
|
||||
print(f"BEST MATCH: {best['device_name']}")
|
||||
print(f"Confidence: {best['confidence']:.1%}")
|
||||
print(f"Frequency Difference: {best['freq_diff_hz']/1e6:.1f} MHz")
|
||||
print(f"{'='*80}")
|
||||
|
||||
else:
|
||||
print("❌ No matches found in database")
|
||||
print("\nReason: T-Embed capture is 915 MHz, but database contains:")
|
||||
|
||||
# Show frequency distribution
|
||||
cursor.execute('''
|
||||
SELECT frequency, COUNT(*) as count
|
||||
FROM signatures
|
||||
GROUP BY frequency
|
||||
''')
|
||||
|
||||
for freq, count in cursor.fetchall():
|
||||
print(f" {freq/1e6:.2f} MHz: {count} devices")
|
||||
|
||||
print("\nTo get a match, need to:")
|
||||
print(" 1. Import RTL_433 signatures (has 915 MHz sensors)")
|
||||
print(" 2. Add more T-Embed wardriving captures")
|
||||
print(" 3. Import community 915 MHz signatures")
|
||||
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("CONCLUSION")
|
||||
print(f"{'='*80}")
|
||||
print("✅ Database populated: 85 devices")
|
||||
print("✅ Matching system: Working")
|
||||
print("❌ Coverage gap: No 915 MHz devices in Flipper database")
|
||||
print("✅ Solution: Import RTL_433 for 915 MHz coverage")
|
||||
print("="*80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Match T-Embed captures against Flipper Zero signature database
|
||||
|
||||
Demonstrates device identification using expanded signature knowledge base
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
|
||||
|
||||
def load_flipper_signatures(flipper_dir: Path) -> List[Dict]:
|
||||
"""Load all Flipper Zero signatures"""
|
||||
|
||||
parser = SubFileParser()
|
||||
signatures = []
|
||||
|
||||
for sub_file in flipper_dir.glob('**/*.sub'):
|
||||
try:
|
||||
metadata = parser.parse(str(sub_file))
|
||||
|
||||
signatures.append({
|
||||
'device_name': sub_file.stem,
|
||||
'filename': sub_file.name,
|
||||
'frequency': metadata.frequency,
|
||||
'protocol': metadata.protocol or 'RAW',
|
||||
'file_format': metadata.file_format,
|
||||
'bit_length': metadata.bit_length,
|
||||
'has_raw': bool(metadata.raw_data),
|
||||
'raw_samples': len(metadata.raw_data) if metadata.raw_data else 0
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
return signatures
|
||||
|
||||
|
||||
def match_by_frequency(target_freq: int, signatures: List[Dict], tolerance_hz: int = 10000) -> List[Dict]:
|
||||
"""Match by frequency with tolerance"""
|
||||
|
||||
matches = []
|
||||
|
||||
for sig in signatures:
|
||||
freq_diff = abs(sig['frequency'] - target_freq)
|
||||
|
||||
if freq_diff <= tolerance_hz:
|
||||
confidence = 1.0 - (freq_diff / tolerance_hz)
|
||||
confidence = max(0.5, confidence)
|
||||
|
||||
matches.append({
|
||||
'signature': sig,
|
||||
'confidence': confidence,
|
||||
'freq_diff_hz': freq_diff,
|
||||
'match_method': 'frequency'
|
||||
})
|
||||
|
||||
return sorted(matches, key=lambda x: x['confidence'], reverse=True)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
|
||||
print("="*80)
|
||||
print("DEVICE MATCHING: T-Embed vs Flipper Zero Database")
|
||||
print("="*80)
|
||||
print()
|
||||
|
||||
# Load Flipper signatures
|
||||
flipper_dir = Path(__file__).parent.parent / 'signatures' / 'flipperzero-firmware'
|
||||
|
||||
if not flipper_dir.exists():
|
||||
print("❌ Flipper Zero database not found")
|
||||
return 1
|
||||
|
||||
print("Loading Flipper Zero signature database...")
|
||||
signatures = load_flipper_signatures(flipper_dir)
|
||||
print(f"✅ Loaded {len(signatures)} signatures\n")
|
||||
|
||||
# Load T-Embed capture
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
parser = SubFileParser()
|
||||
|
||||
tembed_file = tembed_dir / 'raw_7.sub'
|
||||
|
||||
print(f"Analyzing T-Embed capture: {tembed_file.name}")
|
||||
print("-"*80)
|
||||
|
||||
metadata = parser.parse(str(tembed_file))
|
||||
|
||||
print(f"Frequency: {metadata.frequency/1e6:.2f} MHz")
|
||||
print(f"Protocol: {metadata.protocol or 'RAW (undecoded)'}")
|
||||
print(f"Format: {metadata.file_format}")
|
||||
if metadata.raw_data:
|
||||
print(f"RAW Samples: {len(metadata.raw_data)}")
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("MATCHING AGAINST FLIPPER ZERO DATABASE")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Match by frequency (±10 kHz tolerance)
|
||||
matches = match_by_frequency(metadata.frequency, signatures, tolerance_hz=10000)
|
||||
|
||||
if matches:
|
||||
print(f"Found {len(matches)} potential matches:\n")
|
||||
|
||||
for i, match in enumerate(matches[:10], 1):
|
||||
sig = match['signature']
|
||||
print(f"{i}. {sig['device_name']}")
|
||||
print(f" Frequency: {sig['frequency']/1e6:.3f} MHz (diff: {match['freq_diff_hz']/1000:.1f} kHz)")
|
||||
print(f" Protocol: {sig['protocol']}")
|
||||
print(f" Format: {sig['file_format']}")
|
||||
print(f" Confidence: {match['confidence']:.1%}")
|
||||
print()
|
||||
|
||||
# Best match
|
||||
best = matches[0]
|
||||
print(f"{'='*80}")
|
||||
print(f"BEST MATCH: {best['signature']['device_name']}")
|
||||
print(f"Confidence: {best['confidence']:.1%}")
|
||||
print(f"Method: Frequency matching ({best['freq_diff_hz']/1000:.1f} kHz difference)")
|
||||
print(f"{'='*80}")
|
||||
|
||||
else:
|
||||
print("❌ No matches found in Flipper Zero database")
|
||||
print("\nThis device is at 915 MHz (ISM band)")
|
||||
print("Flipper Zero database contains mostly 433 MHz devices")
|
||||
print("\nTo improve matching:")
|
||||
print("- Import RTL_433 database (has 915 MHz devices)")
|
||||
print("- Add more T-Embed captures from wardriving")
|
||||
print("- Import community-contributed 915 MHz signatures")
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("DATABASE COVERAGE ANALYSIS")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Analyze frequency coverage
|
||||
freq_groups = {}
|
||||
for sig in signatures:
|
||||
freq_mhz = sig['frequency'] / 1e6
|
||||
freq_band = f"{int(freq_mhz/100)*100}-{int(freq_mhz/100)*100+100}"
|
||||
|
||||
if freq_band not in freq_groups:
|
||||
freq_groups[freq_band] = 0
|
||||
freq_groups[freq_band] += 1
|
||||
|
||||
print("Frequency Band Coverage:")
|
||||
for band in sorted(freq_groups.keys()):
|
||||
print(f" {band} MHz: {freq_groups[band]} devices")
|
||||
|
||||
# Check if 915 MHz covered
|
||||
target_freq = metadata.frequency / 1e6
|
||||
target_band = f"{int(target_freq/100)*100}-{int(target_freq/100)*100+100}"
|
||||
|
||||
print(f"\nTarget device: {target_freq:.2f} MHz ({target_band} MHz band)")
|
||||
|
||||
if target_band in freq_groups:
|
||||
print(f"✅ Coverage: {freq_groups[target_band]} devices in target band")
|
||||
else:
|
||||
print(f"❌ No coverage: Target band not in Flipper database")
|
||||
|
||||
print("\n" + "="*80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Re-match existing captures with device matcher
|
||||
|
||||
Runs the device matcher on all existing captures in the database
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.matcher.simple_matcher import get_matcher
|
||||
|
||||
|
||||
def rematch_captures(storage_file):
|
||||
"""Re-run device matching on existing captures"""
|
||||
|
||||
if not Path(storage_file).exists():
|
||||
print(f"❌ Storage file not found: {storage_file}")
|
||||
return 1
|
||||
|
||||
# Load existing captures
|
||||
with open(storage_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
captures = data.get('captures', [])
|
||||
|
||||
if not captures:
|
||||
print("ℹ️ No captures to process")
|
||||
return 0
|
||||
|
||||
print(f"🔄 Re-matching {len(captures)} captures...")
|
||||
|
||||
matcher = get_matcher()
|
||||
updated_count = 0
|
||||
|
||||
for i, capture in enumerate(captures, 1):
|
||||
frequency = capture.get('frequency', 0)
|
||||
protocol = capture.get('protocol', 'RAW')
|
||||
preset = capture.get('preset', 'Unknown')
|
||||
|
||||
# Skip if already has device data
|
||||
if capture.get('device_name'):
|
||||
continue
|
||||
|
||||
# Perform matching
|
||||
matches = matcher.match(frequency, protocol, preset)
|
||||
best_match = matches[0] if matches else None
|
||||
|
||||
if best_match:
|
||||
# Update capture with device data
|
||||
capture['device_name'] = best_match.device_name
|
||||
capture['device_category'] = best_match.device_category
|
||||
capture['match_confidence'] = best_match.confidence
|
||||
capture['match_method'] = best_match.match_method
|
||||
capture['device_description'] = best_match.description
|
||||
capture['matched_devices'] = [
|
||||
{
|
||||
"device_name": m.device_name,
|
||||
"category": m.device_category,
|
||||
"confidence": m.confidence,
|
||||
"method": m.match_method,
|
||||
"description": m.description
|
||||
}
|
||||
for m in matches[:5]
|
||||
]
|
||||
updated_count += 1
|
||||
|
||||
print(f" [{i}/{len(captures)}] {capture['filename']}: {best_match.device_name} ({best_match.confidence:.0%})")
|
||||
else:
|
||||
print(f" [{i}/{len(captures)}] {capture['filename']}: No match found")
|
||||
|
||||
# Save updated data
|
||||
with open(storage_file, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
print(f"\n✅ Updated {updated_count}/{len(captures)} captures with device information")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
STORAGE_FILE = Path(__file__).parent.parent / "data" / "captures_simple.json"
|
||||
sys.exit(rematch_captures(STORAGE_FILE))
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test GPS extraction from T-Embed files
|
||||
|
||||
Demonstrates automatic GPS population from filenames
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.gps_extractor import GPSFilenameExtractor
|
||||
|
||||
|
||||
def main():
|
||||
"""Test GPS extraction on T-Embed files"""
|
||||
|
||||
print("=" * 80)
|
||||
print("GPS EXTRACTION TEST - T-Embed Files")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
# Find T-Embed files
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
|
||||
if not tembed_dir.exists():
|
||||
print(f"❌ T-Embed directory not found: {tembed_dir}")
|
||||
return 1
|
||||
|
||||
sub_files = list(tembed_dir.glob('*.sub'))
|
||||
print(f"Found {len(sub_files)} .sub files")
|
||||
print()
|
||||
|
||||
# Test extraction
|
||||
extractor = GPSFilenameExtractor()
|
||||
|
||||
with_gps = []
|
||||
without_gps = []
|
||||
|
||||
for sub_file in sorted(sub_files):
|
||||
coords = extractor.extract(sub_file.name)
|
||||
|
||||
if coords:
|
||||
with_gps.append((sub_file.name, coords))
|
||||
print(f"✅ {sub_file.name}")
|
||||
print(f" 📍 Latitude: {coords.latitude:.6f}")
|
||||
print(f" 📍 Longitude: {coords.longitude:.6f}")
|
||||
print(f" 📋 Source: {coords.source}")
|
||||
else:
|
||||
without_gps.append(sub_file.name)
|
||||
print(f"⏭️ {sub_file.name} - No GPS in filename")
|
||||
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print("=" * 80)
|
||||
print("SUMMARY")
|
||||
print("=" * 80)
|
||||
print(f"Total files: {len(sub_files)}")
|
||||
print(f"With GPS coords: {len(with_gps)} ({len(with_gps)/len(sub_files)*100:.1f}%)")
|
||||
print(f"Without GPS coords: {len(without_gps)} ({len(without_gps)/len(sub_files)*100:.1f}%)")
|
||||
print()
|
||||
|
||||
if with_gps:
|
||||
print("Files with GPS coordinates:")
|
||||
for filename, coords in with_gps:
|
||||
print(f" - {filename}")
|
||||
print(f" {coords.latitude:.6f}, {coords.longitude:.6f}")
|
||||
|
||||
print()
|
||||
|
||||
if without_gps:
|
||||
print("Files without GPS (would need manual entry or JSON companion):")
|
||||
for filename in without_gps:
|
||||
print(f" - {filename}")
|
||||
|
||||
print()
|
||||
print("=" * 80)
|
||||
print("NEXT STEPS")
|
||||
print("=" * 80)
|
||||
print("1. Files with GPS in filename → Auto-populate coordinates")
|
||||
print("2. Files without GPS → Look for companion JSON files")
|
||||
print("3. If no JSON found → Require manual GPS entry during upload")
|
||||
print("=" * 80)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test T-Embed signature matching
|
||||
|
||||
This script:
|
||||
1. Imports T-Embed signatures into database (if not already imported)
|
||||
2. Tests matching engine against the same files
|
||||
3. Shows matching accuracy and confidence scores
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.sub_parser import SubFileParser
|
||||
from src.matcher.strategies_orm import (
|
||||
FrequencyMatcherORM,
|
||||
TimingMatcherORM,
|
||||
RAWPatternMatcherORM,
|
||||
ExactMatcherORM
|
||||
)
|
||||
from src.database.connection import get_session
|
||||
|
||||
|
||||
def test_matching():
|
||||
"""Test signature matching with T-Embed files"""
|
||||
|
||||
print("="*70)
|
||||
print("T-Embed Signature Matching Test")
|
||||
print("="*70)
|
||||
print()
|
||||
|
||||
# Get database session
|
||||
session = get_session()
|
||||
|
||||
# Initialize parser and matchers
|
||||
parser = SubFileParser()
|
||||
|
||||
matchers = [
|
||||
('Exact', ExactMatcherORM(session)),
|
||||
('Frequency', FrequencyMatcherORM(session, tolerance_hz=10000)),
|
||||
('Timing', TimingMatcherORM(session)),
|
||||
('RAW Pattern', RAWPatternMatcherORM(session, min_samples=10))
|
||||
]
|
||||
|
||||
# Find T-Embed files
|
||||
tembed_dir = Path(__file__).parent.parent / 'signatures' / 't-embed-rf'
|
||||
sub_files = sorted(tembed_dir.glob('*.sub'))
|
||||
|
||||
print(f"Found {len(sub_files)} .sub files\n")
|
||||
|
||||
total_files = 0
|
||||
total_matches = 0
|
||||
|
||||
# Test each file
|
||||
for sub_file in sub_files:
|
||||
print("-"*70)
|
||||
print(f"File: {sub_file.name}")
|
||||
|
||||
# Parse file
|
||||
try:
|
||||
metadata = parser.parse(str(sub_file))
|
||||
except Exception as e:
|
||||
print(f" ❌ Parse error: {e}\n")
|
||||
continue
|
||||
|
||||
# Skip empty files
|
||||
if metadata.frequency == 0 or (metadata.file_format == 'RAW' and not metadata.raw_data):
|
||||
print(f" ⏭️ Skipped: Empty capture\n")
|
||||
continue
|
||||
|
||||
total_files += 1
|
||||
|
||||
# Show signal info
|
||||
print(f"\nSignal Info:")
|
||||
print(f" Frequency: {metadata.frequency/1e6:.2f} MHz")
|
||||
print(f" Protocol: {metadata.protocol or 'RAW'}")
|
||||
print(f" Format: {metadata.file_format}")
|
||||
|
||||
if metadata.raw_data:
|
||||
abs_timings = [abs(t) for t in metadata.raw_data]
|
||||
print(f" RAW Samples: {len(metadata.raw_data)}")
|
||||
print(f" Timing Range: {min(abs_timings)}-{max(abs_timings)} μs")
|
||||
print(f" Avg Timing: {sum(abs_timings)/len(abs_timings):.1f} μs")
|
||||
|
||||
# Try each matcher
|
||||
print(f"\nMatching Results:")
|
||||
|
||||
file_matched = False
|
||||
|
||||
for matcher_name, matcher in matchers:
|
||||
try:
|
||||
matches = matcher.match(metadata)
|
||||
|
||||
if matches:
|
||||
file_matched = True
|
||||
print(f"\n {matcher_name} Matcher: {len(matches)} match(es)")
|
||||
|
||||
# Show top 3 matches
|
||||
for i, match in enumerate(matches[:3], 1):
|
||||
print(f" {i}. {match.device_name}")
|
||||
print(f" Manufacturer: {match.manufacturer}")
|
||||
print(f" Confidence: {match.confidence:.2%}")
|
||||
print(f" Method: {match.match_method}")
|
||||
|
||||
# Show match details
|
||||
if match.match_details:
|
||||
for key, value in match.match_details.items():
|
||||
if key != 'signature_id':
|
||||
print(f" {key}: {value}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ {matcher_name} error: {e}")
|
||||
|
||||
if file_matched:
|
||||
total_matches += 1
|
||||
print(f"\n ✅ File matched successfully!")
|
||||
else:
|
||||
print(f"\n ⚠️ No matches found")
|
||||
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print("="*70)
|
||||
print("MATCHING SUMMARY")
|
||||
print("="*70)
|
||||
print(f"Files tested: {total_files}")
|
||||
print(f"Files matched: {total_matches}")
|
||||
|
||||
if total_files > 0:
|
||||
match_rate = (total_matches / total_files) * 100
|
||||
print(f"Match rate: {match_rate:.1f}%")
|
||||
|
||||
if match_rate == 100:
|
||||
print("\n✅ Perfect! All files matched to signatures")
|
||||
elif match_rate >= 75:
|
||||
print(f"\n✅ Good! Most files matched")
|
||||
elif match_rate >= 50:
|
||||
print(f"\n⚠️ Moderate: Some files unmatched")
|
||||
else:
|
||||
print(f"\n❌ Low match rate - may need more signatures or tuning")
|
||||
else:
|
||||
print("\n⚠️ No valid files to test")
|
||||
|
||||
print("="*70)
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_matching()
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Wardriving Data Import
|
||||
|
||||
Tests importing GPS-tagged SubGhz captures from various formats
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.parser.wardriving_importer import (
|
||||
WigleCSVImporter,
|
||||
GPSJSONImporter,
|
||||
BatchImporter
|
||||
)
|
||||
|
||||
|
||||
def test_wigle_csv_import():
|
||||
"""Test Wigle CSV format import"""
|
||||
print("=" * 80)
|
||||
print("TEST 1: Wigle CSV Import")
|
||||
print("=" * 80)
|
||||
|
||||
csv_path = "signatures/t-embed-rf/test_export_wigle.csv"
|
||||
|
||||
if not Path(csv_path).exists():
|
||||
print(f"❌ CSV file not found: {csv_path}")
|
||||
return
|
||||
|
||||
importer = WigleCSVImporter()
|
||||
captures = importer.parse_csv(csv_path)
|
||||
|
||||
print(f"\n✅ Imported {len(captures)} captures from CSV")
|
||||
|
||||
for i, capture in enumerate(captures, 1):
|
||||
print(f"\nCapture {i}:")
|
||||
print(f" Filename: {capture.filename}")
|
||||
print(f" Frequency: {capture.frequency / 1e6:.2f} MHz")
|
||||
print(f" Protocol: {capture.protocol}")
|
||||
print(f" GPS: {capture.latitude}, {capture.longitude}")
|
||||
print(f" Accuracy: {capture.accuracy}m")
|
||||
print(f" Timestamp: {capture.timestamp}")
|
||||
print(f" GPS Source: {capture.gps_source}")
|
||||
|
||||
|
||||
def test_gps_json_import():
|
||||
"""Test GPS JSON format import"""
|
||||
print("\n" + "=" * 80)
|
||||
print("TEST 2: GPS JSON Import")
|
||||
print("=" * 80)
|
||||
|
||||
json_files = list(Path("signatures/t-embed-rf").glob("gps_coordinates_*.json"))
|
||||
|
||||
if not json_files:
|
||||
print("❌ No GPS JSON files found")
|
||||
return
|
||||
|
||||
print(f"\nFound {len(json_files)} GPS JSON files")
|
||||
|
||||
for json_file in json_files[:3]: # Test first 3
|
||||
coords = GPSJSONImporter.parse_json(str(json_file))
|
||||
|
||||
if coords:
|
||||
print(f"\n✅ {json_file.name}")
|
||||
print(f" Latitude: {coords.latitude}")
|
||||
print(f" Longitude: {coords.longitude}")
|
||||
print(f" Accuracy: {coords.accuracy}m")
|
||||
print(f" Source: {coords.source}")
|
||||
else:
|
||||
print(f"\n❌ {json_file.name} - No GPS data")
|
||||
|
||||
|
||||
def test_batch_import():
|
||||
"""Test batch directory import"""
|
||||
print("\n" + "=" * 80)
|
||||
print("TEST 3: Batch Directory Import")
|
||||
print("=" * 80)
|
||||
|
||||
importer = BatchImporter()
|
||||
captures = importer.import_from_directory("signatures/t-embed-rf")
|
||||
|
||||
print(f"\n✅ Imported {len(captures)} total captures")
|
||||
|
||||
# Group by GPS source
|
||||
print("\nGPS Sources:")
|
||||
sources = {}
|
||||
for capture in captures:
|
||||
sources[capture.gps_source] = sources.get(capture.gps_source, 0) + 1
|
||||
|
||||
for source, count in sorted(sources.items()):
|
||||
print(f" {source}: {count}")
|
||||
|
||||
# Show sample captures
|
||||
print("\nSample Captures:")
|
||||
for i, capture in enumerate(captures[:5], 1):
|
||||
print(f"\n {i}. {capture.filename}")
|
||||
print(f" Frequency: {capture.frequency / 1e6:.2f} MHz")
|
||||
print(f" GPS: {capture.latitude}, {capture.longitude}")
|
||||
print(f" Source: {capture.gps_source}")
|
||||
|
||||
|
||||
def test_upload_manifest():
|
||||
"""Test generating upload manifest"""
|
||||
print("\n" + "=" * 80)
|
||||
print("TEST 4: Upload Manifest Generation")
|
||||
print("=" * 80)
|
||||
|
||||
importer = BatchImporter()
|
||||
captures = importer.import_from_directory("signatures/t-embed-rf")
|
||||
|
||||
manifest = importer.to_upload_manifest()
|
||||
|
||||
print(f"\n✅ Generated manifest with {len(manifest['captures'])} captures")
|
||||
print(f"Session UUID: {manifest['session_uuid']}")
|
||||
|
||||
print("\nSample capture data:")
|
||||
if manifest['captures']:
|
||||
import json
|
||||
print(json.dumps(manifest['captures'][0], indent=2))
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all tests"""
|
||||
print("\n🧪 WARDRIVING DATA IMPORT TESTS")
|
||||
print("=" * 80)
|
||||
|
||||
try:
|
||||
test_wigle_csv_import()
|
||||
test_gps_json_import()
|
||||
test_batch_import()
|
||||
test_upload_manifest()
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("✅ ALL TESTS COMPLETE")
|
||||
print("=" * 80)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user