feat: benchmark suite + scoring calibration - iteration 4/5
Implements comprehensive benchmarking infrastructure and tunes scoring weights based on empirical accuracy measurements. New Components: - tests/benchmark/test_data_generator.py: Synthetic signal generator for 12 protocols - scripts/benchmark.py: Full benchmarking suite with accuracy metrics - TEST_RESULTS_SUMMARY.md: Detailed benchmark results and per-protocol analysis Benchmark Results: - Total Tests: 12 synthetic signals across 10 protocols - Top-1 Accuracy: 33.3% (4/12 correct) - Top-3 Accuracy: 33.3% - Confidence Distribution: 66.7% high (>80%), 25% medium (50-80%), 8.3% low (<50%) - Avg Processing Time: 144.6ms per signal Scoring Weight Tuning: BEFORE: Timing(30%) + Frequency(25%) + BitCount(20%) + Preamble(15%) + Stats(10%) AFTER: Timing(35%) + Preamble(25%) + BitCount(20%) + Frequency(15%) + Stats(5%) Rationale: - Increased Preamble weight (15% → 25%): Highly discriminative for protocol identification - Increased Timing weight (30% → 35%): Core identification feature - Decreased Frequency weight (25% → 15%): Many protocols share same ISM band - Decreased Stats weight (10% → 5%): Less discriminative in practice Confidence Thresholds: - High: >80% (reliable identification) - Medium: 50-80% (possible match, needs verification) - Low: <50% (uncertain, likely incorrect) Protocol Performance: ✓ Excellent (100%): LaCrosse TX141-BV2, Oregon Scientific v2.1, Schrader TPMS ✗ Needs Improvement (0%): Acurite 609TXC, Princeton, PT2262, Nexus, Toyota TPMS Key Findings: - Preamble detection critical for discrimination (alternating patterns work well) - Timing analysis robust to 15% noise - 315 MHz protocols underrepresented in database - Generic protocols difficult to distinguish without more specific signatures Next Steps (Future Iterations): - Expand 315 MHz protocol coverage - Add protocol-specific heuristics for Princeton, PT2262 - Improve bit pattern matching for similar timing protocols 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
leetcrypt
@@ -0,0 +1,358 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Synthetic Test Signal Generator for Benchmarking
|
||||
|
||||
Generates .sub files with known protocols for accuracy testing.
|
||||
"""
|
||||
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Tuple
|
||||
|
||||
|
||||
class SyntheticSignalGenerator:
|
||||
"""Generate synthetic RF signals for known protocols"""
|
||||
|
||||
def __init__(self, output_dir: Path):
|
||||
self.output_dir = output_dir
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def generate_pwm_signal(
|
||||
self,
|
||||
protocol_name: str,
|
||||
frequency: int,
|
||||
short_pulse: int,
|
||||
long_pulse: int,
|
||||
short_gap: int,
|
||||
bit_pattern: str,
|
||||
preamble: str = "",
|
||||
noise_level: float = 0.05
|
||||
) -> Path:
|
||||
"""
|
||||
Generate PWM-encoded signal (SHORT=0, LONG=1)
|
||||
|
||||
Args:
|
||||
protocol_name: Protocol name for filename
|
||||
frequency: Center frequency in Hz
|
||||
short_pulse: Short pulse duration (μs)
|
||||
long_pulse: Long pulse duration (μs)
|
||||
short_gap: Gap duration (μs)
|
||||
bit_pattern: Binary pattern to encode
|
||||
preamble: Preamble pattern (optional)
|
||||
noise_level: Jitter as percentage (0.0-1.0)
|
||||
|
||||
Returns:
|
||||
Path to generated .sub file
|
||||
"""
|
||||
pulses = []
|
||||
|
||||
# Add preamble if specified
|
||||
if preamble:
|
||||
for bit in preamble:
|
||||
if bit == '1':
|
||||
pulses.extend([self._jitter(long_pulse, noise_level), -short_gap])
|
||||
else:
|
||||
pulses.extend([self._jitter(short_pulse, noise_level), -short_gap])
|
||||
|
||||
# Encode bit pattern
|
||||
for bit in bit_pattern:
|
||||
if bit == '1':
|
||||
pulses.extend([self._jitter(long_pulse, noise_level), -short_gap])
|
||||
else:
|
||||
pulses.extend([self._jitter(short_pulse, noise_level), -short_gap])
|
||||
|
||||
# Write .sub file
|
||||
filename = f"{protocol_name.lower().replace(' ', '_')}_synthetic.sub"
|
||||
filepath = self.output_dir / filename
|
||||
|
||||
content = f"""Filetype: Flipper SubGhz RAW File
|
||||
Version: 1
|
||||
Frequency: {frequency}
|
||||
Preset: FuriHalSubGhzPresetOok650Async
|
||||
Protocol: RAW
|
||||
RAW_Data: {' '.join(map(str, pulses))}
|
||||
"""
|
||||
|
||||
filepath.write_text(content)
|
||||
return filepath
|
||||
|
||||
def generate_long_burst_signal(
|
||||
self,
|
||||
protocol_name: str,
|
||||
frequency: int,
|
||||
burst_duration: int,
|
||||
short_pulse: int,
|
||||
long_pulse: int,
|
||||
bit_pattern: str,
|
||||
noise_level: float = 0.05
|
||||
) -> Path:
|
||||
"""Generate signal with long burst preamble (Princeton style)"""
|
||||
pulses = []
|
||||
|
||||
# Long burst preamble
|
||||
pulses.append(self._jitter(burst_duration, noise_level))
|
||||
pulses.append(-short_pulse)
|
||||
|
||||
# Data
|
||||
for bit in bit_pattern:
|
||||
if bit == '1':
|
||||
pulses.extend([self._jitter(long_pulse, noise_level), -short_pulse])
|
||||
else:
|
||||
pulses.extend([self._jitter(short_pulse, noise_level), -short_pulse])
|
||||
|
||||
filename = f"{protocol_name.lower().replace(' ', '_')}_synthetic.sub"
|
||||
filepath = self.output_dir / filename
|
||||
|
||||
content = f"""Filetype: Flipper SubGhz RAW File
|
||||
Version: 1
|
||||
Frequency: {frequency}
|
||||
Preset: FuriHalSubGhzPresetOok650Async
|
||||
Protocol: RAW
|
||||
RAW_Data: {' '.join(map(str, pulses))}
|
||||
"""
|
||||
|
||||
filepath.write_text(content)
|
||||
return filepath
|
||||
|
||||
def _jitter(self, value: int, noise_level: float) -> int:
|
||||
"""Add random jitter to pulse duration"""
|
||||
if noise_level == 0:
|
||||
return value
|
||||
jitter = random.gauss(0, noise_level)
|
||||
return int(value * (1 + jitter))
|
||||
|
||||
def generate_test_suite(self) -> List[Tuple[Path, str, Dict]]:
|
||||
"""
|
||||
Generate comprehensive test suite with known protocols
|
||||
|
||||
Returns:
|
||||
List of (filepath, expected_protocol, metadata)
|
||||
"""
|
||||
test_cases = []
|
||||
|
||||
# === Weather Sensors ===
|
||||
|
||||
# 1. LaCrosse TX141-BV2 (433.92 MHz, PWM 500/1000μs)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="LaCrosse TX141-BV2",
|
||||
frequency=433920000,
|
||||
short_pulse=500,
|
||||
long_pulse=1000,
|
||||
short_gap=500,
|
||||
bit_pattern="10101010" + "11001100" * 4, # 40 bits
|
||||
preamble="10101010", # Alternating preamble
|
||||
noise_level=0.05
|
||||
)
|
||||
test_cases.append((filepath, "LaCrosse TX141-BV2", {
|
||||
'category': 'weather_sensor',
|
||||
'frequency': 433920000,
|
||||
'encoding': 'PWM',
|
||||
'timing': '500/1000μs'
|
||||
}))
|
||||
|
||||
# 2. Acurite 609TXC (433.92 MHz, PWM 500/1000μs)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Acurite 609TXC",
|
||||
frequency=433920000,
|
||||
short_pulse=500,
|
||||
long_pulse=1000,
|
||||
short_gap=500,
|
||||
bit_pattern="1100" * 10, # 40 bits
|
||||
preamble="1010",
|
||||
noise_level=0.05
|
||||
)
|
||||
test_cases.append((filepath, "Acurite 609TXC", {
|
||||
'category': 'weather_sensor',
|
||||
'frequency': 433920000
|
||||
}))
|
||||
|
||||
# 3. Oregon Scientific v2.1 (433.92 MHz)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Oregon Scientific v2.1",
|
||||
frequency=433920000,
|
||||
short_pulse=500,
|
||||
long_pulse=1000,
|
||||
short_gap=500,
|
||||
bit_pattern="1000" + "11001010" * 6, # Sync word + data
|
||||
preamble="10101010" * 4,
|
||||
noise_level=0.05
|
||||
)
|
||||
test_cases.append((filepath, "Oregon Scientific v2.1", {
|
||||
'category': 'weather_sensor',
|
||||
'frequency': 433920000
|
||||
}))
|
||||
|
||||
# 4. Nexus-TH (433.92 MHz)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Nexus Temperature-Humidity",
|
||||
frequency=433920000,
|
||||
short_pulse=500,
|
||||
long_pulse=1000,
|
||||
short_gap=500,
|
||||
bit_pattern="11110000" * 5,
|
||||
preamble="1111",
|
||||
noise_level=0.05
|
||||
)
|
||||
test_cases.append((filepath, "Nexus Temperature-Humidity", {
|
||||
'category': 'weather_sensor',
|
||||
'frequency': 433920000
|
||||
}))
|
||||
|
||||
# === Garage Door Openers ===
|
||||
|
||||
# 5. Princeton (315 MHz, long burst preamble)
|
||||
filepath = self.generate_long_burst_signal(
|
||||
protocol_name="Princeton",
|
||||
frequency=315000000,
|
||||
burst_duration=4000, # 4ms burst
|
||||
short_pulse=400,
|
||||
long_pulse=1200,
|
||||
bit_pattern="110101101001" * 2, # 24 bits
|
||||
noise_level=0.05
|
||||
)
|
||||
test_cases.append((filepath, "Princeton", {
|
||||
'category': 'garage_door',
|
||||
'frequency': 315000000,
|
||||
'preamble_type': 'long_burst'
|
||||
}))
|
||||
|
||||
# 6. PT2262 (433.92 MHz, long burst)
|
||||
filepath = self.generate_long_burst_signal(
|
||||
protocol_name="PT2262",
|
||||
frequency=433920000,
|
||||
burst_duration=3200,
|
||||
short_pulse=350,
|
||||
long_pulse=1050,
|
||||
bit_pattern="1111000011110000" + "10101010",
|
||||
noise_level=0.05
|
||||
)
|
||||
test_cases.append((filepath, "PT2262", {
|
||||
'category': 'remote_control',
|
||||
'frequency': 433920000
|
||||
}))
|
||||
|
||||
# === Tire Pressure Sensors ===
|
||||
|
||||
# 7. Schrader TPMS (315 MHz)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Schrader TPMS",
|
||||
frequency=315000000,
|
||||
short_pulse=100,
|
||||
long_pulse=200,
|
||||
short_gap=100,
|
||||
bit_pattern="10101100" * 8, # 64 bits
|
||||
preamble="10101010" * 2,
|
||||
noise_level=0.03
|
||||
)
|
||||
test_cases.append((filepath, "Schrader TPMS", {
|
||||
'category': 'tire_pressure',
|
||||
'frequency': 315000000
|
||||
}))
|
||||
|
||||
# 8. Toyota TPMS (315 MHz)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Toyota TPMS",
|
||||
frequency=315000000,
|
||||
short_pulse=100,
|
||||
long_pulse=200,
|
||||
short_gap=100,
|
||||
bit_pattern="11001100" * 9, # 72 bits
|
||||
preamble="1111000011110000",
|
||||
noise_level=0.03
|
||||
)
|
||||
test_cases.append((filepath, "Toyota TPMS", {
|
||||
'category': 'tire_pressure',
|
||||
'frequency': 315000000
|
||||
}))
|
||||
|
||||
# === Security Sensors (868 MHz) ===
|
||||
|
||||
# 9. Honeywell Security (868 MHz)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Honeywell Security",
|
||||
frequency=868000000,
|
||||
short_pulse=250,
|
||||
long_pulse=750,
|
||||
short_gap=250,
|
||||
bit_pattern="10110011" * 6,
|
||||
preamble="1010" * 4,
|
||||
noise_level=0.04
|
||||
)
|
||||
test_cases.append((filepath, "Honeywell Security", {
|
||||
'category': 'security',
|
||||
'frequency': 868000000
|
||||
}))
|
||||
|
||||
# === Doorbells ===
|
||||
|
||||
# 10. Generic Doorbell (433.92 MHz)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="Generic Doorbell",
|
||||
frequency=433920000,
|
||||
short_pulse=300,
|
||||
long_pulse=900,
|
||||
short_gap=300,
|
||||
bit_pattern="110011001100" * 2,
|
||||
preamble="1111",
|
||||
noise_level=0.06
|
||||
)
|
||||
test_cases.append((filepath, "Generic Doorbell", {
|
||||
'category': 'doorbell',
|
||||
'frequency': 433920000
|
||||
}))
|
||||
|
||||
# === Noisy Signals (High Jitter) ===
|
||||
|
||||
# 11. Noisy LaCrosse (test noise tolerance)
|
||||
filepath = self.generate_pwm_signal(
|
||||
protocol_name="LaCrosse TX141-BV2 Noisy",
|
||||
frequency=433920000,
|
||||
short_pulse=500,
|
||||
long_pulse=1000,
|
||||
short_gap=500,
|
||||
bit_pattern="10101010" + "11001100" * 4,
|
||||
preamble="10101010",
|
||||
noise_level=0.15 # 15% jitter
|
||||
)
|
||||
test_cases.append((filepath, "LaCrosse TX141-BV2", {
|
||||
'category': 'weather_sensor',
|
||||
'frequency': 433920000,
|
||||
'noise': 'high'
|
||||
}))
|
||||
|
||||
# 12. Noisy Princeton
|
||||
filepath = self.generate_long_burst_signal(
|
||||
protocol_name="Princeton Noisy",
|
||||
frequency=315000000,
|
||||
burst_duration=4000,
|
||||
short_pulse=400,
|
||||
long_pulse=1200,
|
||||
bit_pattern="110101101001" * 2,
|
||||
noise_level=0.12
|
||||
)
|
||||
test_cases.append((filepath, "Princeton", {
|
||||
'category': 'garage_door',
|
||||
'frequency': 315000000,
|
||||
'noise': 'high'
|
||||
}))
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Generate test suite
|
||||
output_dir = Path(__file__).parent / "synthetic_signals"
|
||||
generator = SyntheticSignalGenerator(output_dir)
|
||||
|
||||
print("=== Generating Synthetic Test Signals ===")
|
||||
print()
|
||||
|
||||
test_cases = generator.generate_test_suite()
|
||||
|
||||
print(f"Generated {len(test_cases)} test signals:")
|
||||
for filepath, expected_protocol, metadata in test_cases:
|
||||
print(f" ✓ {filepath.name} → {expected_protocol}")
|
||||
|
||||
print()
|
||||
print(f"Output directory: {output_dir}")
|
||||
print(f"Total files: {len(list(output_dir.glob('*.sub')))}")
|
||||
Reference in New Issue
Block a user