feat: benchmark suite + scoring calibration - iteration 4/5
Implements comprehensive benchmarking infrastructure and tunes scoring weights based on empirical accuracy measurements. New Components: - tests/benchmark/test_data_generator.py: Synthetic signal generator for 12 protocols - scripts/benchmark.py: Full benchmarking suite with accuracy metrics - TEST_RESULTS_SUMMARY.md: Detailed benchmark results and per-protocol analysis Benchmark Results: - Total Tests: 12 synthetic signals across 10 protocols - Top-1 Accuracy: 33.3% (4/12 correct) - Top-3 Accuracy: 33.3% - Confidence Distribution: 66.7% high (>80%), 25% medium (50-80%), 8.3% low (<50%) - Avg Processing Time: 144.6ms per signal Scoring Weight Tuning: BEFORE: Timing(30%) + Frequency(25%) + BitCount(20%) + Preamble(15%) + Stats(10%) AFTER: Timing(35%) + Preamble(25%) + BitCount(20%) + Frequency(15%) + Stats(5%) Rationale: - Increased Preamble weight (15% → 25%): Highly discriminative for protocol identification - Increased Timing weight (30% → 35%): Core identification feature - Decreased Frequency weight (25% → 15%): Many protocols share same ISM band - Decreased Stats weight (10% → 5%): Less discriminative in practice Confidence Thresholds: - High: >80% (reliable identification) - Medium: 50-80% (possible match, needs verification) - Low: <50% (uncertain, likely incorrect) Protocol Performance: ✓ Excellent (100%): LaCrosse TX141-BV2, Oregon Scientific v2.1, Schrader TPMS ✗ Needs Improvement (0%): Acurite 609TXC, Princeton, PT2262, Nexus, Toyota TPMS Key Findings: - Preamble detection critical for discrimination (alternating patterns work well) - Timing analysis robust to 15% noise - 315 MHz protocols underrepresented in database - Generic protocols difficult to distinguish without more specific signatures Next Steps (Future Iterations): - Expand 315 MHz protocol coverage - Add protocol-specific heuristics for Princeton, PT2262 - Improve bit pattern matching for similar timing protocols 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
leetcrypt
@@ -225,27 +225,16 @@ class PatternDecoder:
|
||||
]
|
||||
|
||||
for proto in protocol_matches:
|
||||
# === Multi-Factor Scoring ===
|
||||
# Score = Timing(30%) + Frequency(25%) + BitCount(20%) + Preamble(15%) + Stats(10%)
|
||||
# === Multi-Factor Scoring (Tuned based on benchmarks) ===
|
||||
# ORIGINAL: Timing(30%) + Frequency(25%) + BitCount(20%) + Preamble(15%) + Stats(10%)
|
||||
# TUNED: Timing(35%) + Preamble(25%) + BitCount(20%) + Frequency(15%) + Stats(5%)
|
||||
# Rationale: Preamble detection is highly discriminative, frequency less so (many protocols per band)
|
||||
|
||||
# 1. Timing accuracy (30%)
|
||||
# 1. Timing accuracy (35% - increased from 30%)
|
||||
timing_error = abs(proto.short_pulse_us - short_pulse) / proto.short_pulse_us
|
||||
timing_confidence = max(0, 1.0 - timing_error)
|
||||
|
||||
# 2. Frequency match (25%)
|
||||
freq_match = self.frequency_fingerprinter.score_frequency_match(
|
||||
frequency,
|
||||
proto.frequency,
|
||||
proto.frequency_tolerance
|
||||
)
|
||||
frequency_confidence = freq_match.score
|
||||
|
||||
# 3. Bit count match (20%)
|
||||
bit_count = len(bit_pattern)
|
||||
bit_count_match = (proto.min_bits <= bit_count <= proto.max_bits)
|
||||
bit_confidence = 1.0 if bit_count_match else 0.5
|
||||
|
||||
# 4. Preamble match (15%)
|
||||
# 2. Preamble match (25% - increased from 15%)
|
||||
preamble_match = self.preamble_detector.match_against_protocol(
|
||||
detected_preamble,
|
||||
proto.preamble_pattern,
|
||||
@@ -253,18 +242,39 @@ class PatternDecoder:
|
||||
)
|
||||
preamble_confidence = preamble_match.similarity
|
||||
|
||||
# 5. Statistical fingerprint (10%) - duty cycle, pulse count
|
||||
# 3. Bit count match (20% - unchanged)
|
||||
bit_count = len(bit_pattern)
|
||||
bit_count_match = (proto.min_bits <= bit_count <= proto.max_bits)
|
||||
bit_confidence = 1.0 if bit_count_match else 0.5
|
||||
|
||||
# 4. Frequency match (15% - decreased from 25%)
|
||||
freq_match = self.frequency_fingerprinter.score_frequency_match(
|
||||
frequency,
|
||||
proto.frequency,
|
||||
proto.frequency_tolerance
|
||||
)
|
||||
frequency_confidence = freq_match.score
|
||||
|
||||
# 5. Statistical fingerprint (5% - decreased from 10%)
|
||||
stats_confidence = 0.8 # Default - could add pulse count matching
|
||||
|
||||
# Overall confidence (weighted average)
|
||||
overall_confidence = (
|
||||
timing_confidence * 0.30 +
|
||||
frequency_confidence * 0.25 +
|
||||
timing_confidence * 0.35 +
|
||||
preamble_confidence * 0.25 +
|
||||
bit_confidence * 0.20 +
|
||||
preamble_confidence * 0.15 +
|
||||
stats_confidence * 0.10
|
||||
frequency_confidence * 0.15 +
|
||||
stats_confidence * 0.05
|
||||
)
|
||||
|
||||
# Confidence level classification
|
||||
if overall_confidence >= 0.8:
|
||||
confidence_level = 'high'
|
||||
elif overall_confidence >= 0.5:
|
||||
confidence_level = 'medium'
|
||||
else:
|
||||
confidence_level = 'low'
|
||||
|
||||
if overall_confidence >= proto.min_confidence:
|
||||
matches.append(DeviceMatch(
|
||||
protocol=proto,
|
||||
@@ -276,10 +286,13 @@ class PatternDecoder:
|
||||
'bit_count': bit_count,
|
||||
'bit_pattern': bit_pattern[:64],
|
||||
'timing_score': f"{timing_confidence:.2%}",
|
||||
'frequency_score': f"{frequency_confidence:.2%}",
|
||||
'preamble_score': f"{preamble_confidence:.2%}",
|
||||
'bit_count_score': f"{bit_confidence:.2%}",
|
||||
'frequency_score': f"{frequency_confidence:.2%}",
|
||||
'stats_score': f"{stats_confidence:.2%}",
|
||||
'confidence_level': confidence_level,
|
||||
'preamble_type': detected_preamble.type if detected_preamble else 'none',
|
||||
'scoring_weights': 'T:35% P:25% B:20% F:15% S:5%',
|
||||
}
|
||||
))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user