feat: Phase 2 & 3 - RTL_433 integration + RAW timing analysis
Phase 2: RTL_433 Protocol Matcher (286 devices) ================================================ Created src/matcher/rtl433_matcher.py - RTL433Matcher class with JSON database loader - Built search indexes: device_id, name, category, modulation - Fuzzy matching with difflib.SequenceMatcher (>0.6 similarity) - Timing signature matching (±15% tolerance) - Confidence scoring: * Exact ID match: 0.95 * Exact name match: 0.90 * Fuzzy match: 0.70-0.85 * Timing match: 0.70-0.95 - Singleton pattern for performance Phase 3: RAW Signal Timing Analysis ==================================== Created src/parser/raw_parser.py - RAWParser class for Flipper Zero RAW_Data format - TimingSignature dataclass with pulse analysis - Extracts short_pulse, long_pulse, gap, pulse_ratio - Percentile-based clustering (25th/75th) - Encoding detection (PWM, PPM, Manchester, OOK) - Statistical analysis (mean, std, total duration) Integration & Enhancements =========================== Enhanced src/matcher/simple_matcher.py - Added raw_data parameter to match() method - RTL_433 protocol matching (Phase 2) with logging - Timing analysis for RAW captures (Phase 3) - Graceful degradation with try/except - RTL433_AVAILABLE flag for feature detection - Maintains backward compatibility Updated src/api/main_simple.py - Extract raw_data from parsed metadata - Convert List[int] to space-separated string - Pass raw_data to enhanced matcher Validation Results ================== Test script: test_enhanced_matcher.py - 20 existing captures re-matched - 4 captures improved (20%) - 0 captures worse (0%) - Average improvement: +0.16 confidence - Best improvement: +0.35 (MegaCode → Linear Megacode) - RTL_433 exact match: MegaCode → 0.95 confidence - RTL_433 fuzzy match: Princeton → Insteon 0.79 Expected Accuracy ================= - Phase 2 alone: 75-80% (+15%) - Phase 2 + 3: 80-85% (+20-25%) - Current validation: Phase 2 confirmed working - Phase 3: Requires new uploads with raw_data Deployment Ready ================ - Backward compatible (optional raw_data) - No breaking changes to API - Graceful import fallback - Ready for server deployment
This commit is contained in:
@@ -0,0 +1,352 @@
|
||||
"""
|
||||
RTL_433 Device Matcher
|
||||
|
||||
Loads and indexes the RTL_433 protocol database for high-accuracy device matching.
|
||||
Provides fuzzy name matching, timing-based matching, and category lookups.
|
||||
|
||||
Author: GigLez Team
|
||||
Date: January 2026
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from dataclasses import dataclass
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RTL433Match:
|
||||
"""Match result from RTL_433 database"""
|
||||
device_id: str
|
||||
device_name: str
|
||||
category: str
|
||||
manufacturer: Optional[str]
|
||||
modulation: Optional[str]
|
||||
confidence: float
|
||||
match_method: str
|
||||
timing_data: Optional[Dict] = None
|
||||
|
||||
|
||||
class RTL433Matcher:
|
||||
"""
|
||||
High-performance matcher using RTL_433 protocol database
|
||||
|
||||
Features:
|
||||
- Fast device ID and name lookups
|
||||
- Fuzzy name matching for partial matches
|
||||
- Modulation-based filtering
|
||||
- Timing signature matching
|
||||
- Category-based fallbacks
|
||||
"""
|
||||
|
||||
def __init__(self, database_path: str = "data/rtl_433_protocols.json"):
|
||||
"""
|
||||
Initialize RTL_433 matcher with protocol database
|
||||
|
||||
Args:
|
||||
database_path: Path to rtl_433_protocols.json
|
||||
"""
|
||||
self.database_path = Path(database_path)
|
||||
self.devices: List[Dict] = []
|
||||
self.device_by_id: Dict[str, Dict] = {}
|
||||
self.device_by_name: Dict[str, Dict] = {}
|
||||
self.devices_by_category: Dict[str, List[Dict]] = {}
|
||||
self.devices_by_modulation: Dict[str, List[Dict]] = {}
|
||||
|
||||
self._load_database()
|
||||
self._build_indexes()
|
||||
|
||||
logger.info(f"RTL_433 matcher initialized with {len(self.devices)} devices")
|
||||
|
||||
def _load_database(self):
|
||||
"""Load RTL_433 protocol database from JSON"""
|
||||
try:
|
||||
if not self.database_path.exists():
|
||||
logger.warning(f"RTL_433 database not found: {self.database_path}")
|
||||
return
|
||||
|
||||
with open(self.database_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
self.devices = data.get('devices', [])
|
||||
|
||||
logger.info(f"Loaded {len(self.devices)} RTL_433 protocols")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load RTL_433 database: {e}")
|
||||
self.devices = []
|
||||
|
||||
def _build_indexes(self):
|
||||
"""Build search indexes for fast lookups"""
|
||||
for device in self.devices:
|
||||
# Index by device ID
|
||||
device_id = device.get('device_id', '').lower()
|
||||
if device_id:
|
||||
self.device_by_id[device_id] = device
|
||||
|
||||
# Index by device name (lowercase for case-insensitive)
|
||||
device_name = device.get('name', '').lower()
|
||||
if device_name:
|
||||
self.device_by_name[device_name] = device
|
||||
|
||||
# Index by category
|
||||
category = device.get('category', 'other')
|
||||
if category not in self.devices_by_category:
|
||||
self.devices_by_category[category] = []
|
||||
self.devices_by_category[category].append(device)
|
||||
|
||||
# Index by modulation
|
||||
modulation = device.get('modulation')
|
||||
if modulation:
|
||||
if modulation not in self.devices_by_modulation:
|
||||
self.devices_by_modulation[modulation] = []
|
||||
self.devices_by_modulation[modulation].append(device)
|
||||
|
||||
logger.info(f"Built indexes: {len(self.device_by_id)} IDs, "
|
||||
f"{len(self.devices_by_category)} categories, "
|
||||
f"{len(self.devices_by_modulation)} modulations")
|
||||
|
||||
def match_by_protocol_name(self, protocol: str) -> List[RTL433Match]:
|
||||
"""
|
||||
Match by protocol/device name
|
||||
|
||||
Tries:
|
||||
1. Exact device ID match
|
||||
2. Exact device name match
|
||||
3. Fuzzy name match (similarity > 0.6)
|
||||
|
||||
Args:
|
||||
protocol: Protocol name from .sub file (e.g., "acurite_rain_896", "Oregon")
|
||||
|
||||
Returns:
|
||||
List of matches sorted by confidence
|
||||
"""
|
||||
matches = []
|
||||
protocol_lower = protocol.lower()
|
||||
|
||||
# 1. Exact device ID match
|
||||
if protocol_lower in self.device_by_id:
|
||||
device = self.device_by_id[protocol_lower]
|
||||
matches.append(RTL433Match(
|
||||
device_id=device['device_id'],
|
||||
device_name=device['name'],
|
||||
category=device.get('category', 'other'),
|
||||
manufacturer=device.get('manufacturer'),
|
||||
modulation=device.get('modulation'),
|
||||
confidence=0.95,
|
||||
match_method='rtl433_exact_id'
|
||||
))
|
||||
return matches
|
||||
|
||||
# 2. Exact device name match
|
||||
if protocol_lower in self.device_by_name:
|
||||
device = self.device_by_name[protocol_lower]
|
||||
matches.append(RTL433Match(
|
||||
device_id=device['device_id'],
|
||||
device_name=device['name'],
|
||||
category=device.get('category', 'other'),
|
||||
manufacturer=device.get('manufacturer'),
|
||||
modulation=device.get('modulation'),
|
||||
confidence=0.90,
|
||||
match_method='rtl433_exact_name'
|
||||
))
|
||||
return matches
|
||||
|
||||
# 3. Fuzzy matching - check all devices
|
||||
for device in self.devices:
|
||||
device_id = device.get('device_id', '').lower()
|
||||
device_name = device.get('name', '').lower()
|
||||
|
||||
# Calculate similarity scores
|
||||
id_similarity = SequenceMatcher(None, protocol_lower, device_id).ratio()
|
||||
name_similarity = SequenceMatcher(None, protocol_lower, device_name).ratio()
|
||||
|
||||
# Also check if protocol is substring
|
||||
substring_match = protocol_lower in device_id or protocol_lower in device_name
|
||||
|
||||
# Take best similarity
|
||||
similarity = max(id_similarity, name_similarity)
|
||||
if substring_match:
|
||||
similarity = max(similarity, 0.7)
|
||||
|
||||
if similarity > 0.6:
|
||||
matches.append(RTL433Match(
|
||||
device_id=device['device_id'],
|
||||
device_name=device['name'],
|
||||
category=device.get('category', 'other'),
|
||||
manufacturer=device.get('manufacturer'),
|
||||
modulation=device.get('modulation'),
|
||||
confidence=0.70 + (similarity * 0.15), # 0.70-0.85 range
|
||||
match_method='rtl433_fuzzy'
|
||||
))
|
||||
|
||||
# Sort by confidence
|
||||
matches.sort(key=lambda x: x.confidence, reverse=True)
|
||||
return matches[:5] # Top 5 matches
|
||||
|
||||
def match_by_modulation(self, modulation: str, category: Optional[str] = None) -> List[RTL433Match]:
|
||||
"""
|
||||
Match devices by modulation type
|
||||
|
||||
Args:
|
||||
modulation: Modulation type (OOK, FSK, etc.)
|
||||
category: Optional category filter
|
||||
|
||||
Returns:
|
||||
List of matches
|
||||
"""
|
||||
matches = []
|
||||
|
||||
devices = self.devices_by_modulation.get(modulation, [])
|
||||
|
||||
# Filter by category if provided
|
||||
if category:
|
||||
devices = [d for d in devices if d.get('category') == category]
|
||||
|
||||
# Return top matches
|
||||
for device in devices[:10]:
|
||||
matches.append(RTL433Match(
|
||||
device_id=device['device_id'],
|
||||
device_name=device['name'],
|
||||
category=device.get('category', 'other'),
|
||||
manufacturer=device.get('manufacturer'),
|
||||
modulation=device.get('modulation'),
|
||||
confidence=0.55, # Lower confidence for modulation-only match
|
||||
match_method='rtl433_modulation'
|
||||
))
|
||||
|
||||
return matches
|
||||
|
||||
def match_by_timing(self, short_pulse: int, long_pulse: int,
|
||||
gap_limit: Optional[int] = None,
|
||||
tolerance: float = 0.15) -> List[RTL433Match]:
|
||||
"""
|
||||
Match devices by timing signature
|
||||
|
||||
Args:
|
||||
short_pulse: Short pulse width in microseconds
|
||||
long_pulse: Long pulse width in microseconds
|
||||
gap_limit: Gap limit in microseconds (optional)
|
||||
tolerance: Matching tolerance (default 15%)
|
||||
|
||||
Returns:
|
||||
List of matches sorted by timing similarity
|
||||
"""
|
||||
matches = []
|
||||
|
||||
for device in self.devices:
|
||||
device_short = device.get('short_width')
|
||||
device_long = device.get('long_width')
|
||||
device_gap = device.get('gap_limit')
|
||||
|
||||
if not (device_short and device_long):
|
||||
continue
|
||||
|
||||
# Calculate timing similarity
|
||||
short_diff = abs(short_pulse - device_short) / device_short
|
||||
long_diff = abs(long_pulse - device_long) / device_long
|
||||
|
||||
# Both must be within tolerance
|
||||
if short_diff <= tolerance and long_diff <= tolerance:
|
||||
# Calculate confidence based on closeness
|
||||
similarity = 1.0 - ((short_diff + long_diff) / 2)
|
||||
|
||||
# Bonus for gap match
|
||||
gap_bonus = 0.0
|
||||
if gap_limit and device_gap:
|
||||
gap_diff = abs(gap_limit - device_gap) / device_gap
|
||||
if gap_diff <= tolerance:
|
||||
gap_bonus = 0.05
|
||||
|
||||
confidence = 0.70 + (similarity * 0.20) + gap_bonus
|
||||
|
||||
matches.append(RTL433Match(
|
||||
device_id=device['device_id'],
|
||||
device_name=device['name'],
|
||||
category=device.get('category', 'other'),
|
||||
manufacturer=device.get('manufacturer'),
|
||||
modulation=device.get('modulation'),
|
||||
confidence=min(confidence, 0.95),
|
||||
match_method='rtl433_timing',
|
||||
timing_data={
|
||||
'short_pulse': device_short,
|
||||
'long_pulse': device_long,
|
||||
'gap_limit': device_gap,
|
||||
'similarity': similarity
|
||||
}
|
||||
))
|
||||
|
||||
# Sort by confidence
|
||||
matches.sort(key=lambda x: x.confidence, reverse=True)
|
||||
return matches[:5]
|
||||
|
||||
def get_devices_by_category(self, category: str) -> List[Dict]:
|
||||
"""Get all devices in a category"""
|
||||
return self.devices_by_category.get(category, [])
|
||||
|
||||
def get_statistics(self) -> Dict:
|
||||
"""Get database statistics"""
|
||||
return {
|
||||
'total_devices': len(self.devices),
|
||||
'categories': {cat: len(devs) for cat, devs in self.devices_by_category.items()},
|
||||
'modulations': {mod: len(devs) for mod, devs in self.devices_by_modulation.items()},
|
||||
'manufacturers': len(set(d.get('manufacturer') for d in self.devices if d.get('manufacturer')))
|
||||
}
|
||||
|
||||
|
||||
# Global singleton instance
|
||||
_rtl433_matcher: Optional[RTL433Matcher] = None
|
||||
|
||||
|
||||
def get_rtl433_matcher() -> RTL433Matcher:
|
||||
"""Get or create RTL_433 matcher singleton"""
|
||||
global _rtl433_matcher
|
||||
if _rtl433_matcher is None:
|
||||
_rtl433_matcher = RTL433Matcher()
|
||||
return _rtl433_matcher
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Test the matcher
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
matcher = RTL433Matcher()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("RTL_433 MATCHER TEST")
|
||||
print("="*60)
|
||||
|
||||
# Test 1: Exact match
|
||||
print("\n1. Exact Protocol Match: 'acurite_rain_896'")
|
||||
matches = matcher.match_by_protocol_name('acurite_rain_896')
|
||||
for match in matches:
|
||||
print(f" {match.device_name} ({match.confidence:.2f}) - {match.match_method}")
|
||||
|
||||
# Test 2: Fuzzy match
|
||||
print("\n2. Fuzzy Match: 'Oregon'")
|
||||
matches = matcher.match_by_protocol_name('Oregon')
|
||||
for match in matches:
|
||||
print(f" {match.device_name} ({match.confidence:.2f}) - {match.match_method}")
|
||||
|
||||
# Test 3: Modulation match
|
||||
print("\n3. Modulation Match: 'OOK'")
|
||||
matches = matcher.match_by_modulation('OOK', category='weather')[:3]
|
||||
for match in matches:
|
||||
print(f" {match.device_name} ({match.confidence:.2f})")
|
||||
|
||||
# Test 4: Timing match
|
||||
print("\n4. Timing Match: short=1000µs, long=2000µs")
|
||||
matches = matcher.match_by_timing(1000, 2000)
|
||||
for match in matches:
|
||||
print(f" {match.device_name} ({match.confidence:.2f})")
|
||||
|
||||
# Statistics
|
||||
print("\n" + "="*60)
|
||||
stats = matcher.get_statistics()
|
||||
print(f"Total devices: {stats['total_devices']}")
|
||||
print(f"Categories: {len(stats['categories'])}")
|
||||
print(f"Modulations: {len(stats['modulations'])}")
|
||||
print("="*60)
|
||||
@@ -1,12 +1,28 @@
|
||||
"""
|
||||
Simple Device Matcher - Frequency-based categorization without database
|
||||
Enhanced Device Matcher - RTL_433 + Timing Analysis + Frequency-based categorization
|
||||
|
||||
Uses frequency + protocol to infer likely device types based on industry standards
|
||||
Integrates:
|
||||
- RTL_433 protocol database (286 devices)
|
||||
- RAW signal timing analysis
|
||||
- Frequency-based categorization
|
||||
- Protocol pattern matching
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Dict, Tuple, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Import enhanced matchers
|
||||
try:
|
||||
from src.matcher.rtl433_matcher import get_rtl433_matcher, RTL433Match
|
||||
from src.parser.raw_parser import parse_raw_data
|
||||
RTL433_AVAILABLE = True
|
||||
except ImportError:
|
||||
RTL433_AVAILABLE = False
|
||||
logging.warning("RTL_433 matcher not available, using fallback matching")
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeviceMatch:
|
||||
@@ -146,21 +162,71 @@ class SimpleDeviceMatcher:
|
||||
],
|
||||
}
|
||||
|
||||
def match(self, frequency: int, protocol: str = None, preset: str = None) -> List[DeviceMatch]:
|
||||
def match(self, frequency: int, protocol: str = None, preset: str = None,
|
||||
raw_data: str = None) -> List[DeviceMatch]:
|
||||
"""
|
||||
Match device based on frequency, protocol, and modulation
|
||||
Enhanced device matching with RTL_433 and timing analysis
|
||||
|
||||
Args:
|
||||
frequency: Frequency in Hz
|
||||
protocol: Protocol name (e.g., "Princeton", "RAW")
|
||||
preset: Preset/modulation (e.g., "FuriHalSubGhzPresetOok270Async")
|
||||
raw_data: RAW_Data string for timing analysis (optional)
|
||||
|
||||
Returns:
|
||||
List of DeviceMatch objects sorted by confidence
|
||||
"""
|
||||
matches = []
|
||||
|
||||
# 1. Protocol-based matching (highest confidence)
|
||||
# PHASE 2: RTL_433 Protocol Database Matching (NEW!)
|
||||
if RTL433_AVAILABLE and protocol and protocol != "RAW":
|
||||
try:
|
||||
rtl433_matcher = get_rtl433_matcher()
|
||||
rtl433_matches = rtl433_matcher.match_by_protocol_name(protocol)
|
||||
|
||||
for rtl_match in rtl433_matches:
|
||||
matches.append(DeviceMatch(
|
||||
device_name=rtl_match.device_name,
|
||||
device_category=rtl_match.category,
|
||||
confidence=rtl_match.confidence,
|
||||
match_method=rtl_match.match_method,
|
||||
description=f"{rtl_match.manufacturer or 'Unknown'} - {rtl_match.modulation or 'Unknown'} modulation"
|
||||
))
|
||||
logger.info(f"RTL_433 match: {rtl_match.device_name} ({rtl_match.confidence:.2f})")
|
||||
except Exception as e:
|
||||
logger.warning(f"RTL_433 matching failed: {e}")
|
||||
|
||||
# PHASE 3: Timing Analysis for RAW captures (NEW!)
|
||||
if RTL433_AVAILABLE and raw_data and protocol == "RAW":
|
||||
try:
|
||||
# Parse RAW signal
|
||||
timing_sig = parse_raw_data(raw_data)
|
||||
if timing_sig:
|
||||
logger.info(f"Timing signature: {timing_sig.short_pulse}µs/{timing_sig.long_pulse}µs, "
|
||||
f"encoding={timing_sig.encoding_type}")
|
||||
|
||||
# Match against RTL_433 timing signatures
|
||||
rtl433_matcher = get_rtl433_matcher()
|
||||
timing_matches = rtl433_matcher.match_by_timing(
|
||||
timing_sig.short_pulse,
|
||||
timing_sig.long_pulse,
|
||||
timing_sig.gap,
|
||||
tolerance=0.15
|
||||
)
|
||||
|
||||
for rtl_match in timing_matches:
|
||||
matches.append(DeviceMatch(
|
||||
device_name=rtl_match.device_name,
|
||||
device_category=rtl_match.category,
|
||||
confidence=rtl_match.confidence,
|
||||
match_method=rtl_match.match_method,
|
||||
description=f"Timing match: {rtl_match.timing_data['similarity']:.2%} similarity"
|
||||
))
|
||||
logger.info(f"Timing match: {rtl_match.device_name} ({rtl_match.confidence:.2f})")
|
||||
except Exception as e:
|
||||
logger.warning(f"Timing analysis failed: {e}")
|
||||
|
||||
# 1. Protocol-based matching (original, lower confidence)
|
||||
if protocol and protocol != "RAW":
|
||||
protocol_matches = self._match_by_protocol(protocol)
|
||||
matches.extend(protocol_matches)
|
||||
|
||||
Reference in New Issue
Block a user