#!/usr/bin/env python3 """ Phase 0 Benchmark — Real-World Signal Test Suite Generates synthetic-but-realistic .sub files matching the timing parameters of real devices from REAL_TEST_RESULTS.md, then tests the Phase 0 matcher. Baseline: 0% top-1, 0% top-3 (from REAL_TEST_RESULTS.md) Target: ≥ 30% top-3 accuracy """ import sys import time import json import tempfile import os from pathlib import Path from dataclasses import dataclass, field from typing import List, Optional sys.path.insert(0, str(Path(__file__).parent.parent)) from src.parser.sub_parser import parse_sub_file from src.matcher.pattern_decoder import get_pattern_decoder # ── Test case definitions ────────────────────────────────────────────────── # Parameters derived from RTL_433 protocol definitions and Flipper Zero firmware. # Each case includes the expected top-1 match OR the expected category. @dataclass class TestCase: name: str expected_device: str # Exact device name or partial string expected_category: str # Acceptable category if exact match fails frequency: int short_pulse_us: int long_pulse_us: int n_bits: int repeats: int = 3 preamble_pulses: int = 0 # Alternating preamble pulses before data sync_gap_us: int = 0 # Sync gap (negative/long gap before data) REAL_WORLD_TEST_CASES = [ # ── Weather sensors ──────────────────────────────────────────────────── TestCase( name="lacrosse_tx141_real", expected_device="LaCrosse", expected_category="Weather Sensor", frequency=433_920_000, short_pulse_us=500, long_pulse_us=1000, n_bits=40, preamble_pulses=8, ), TestCase( name="acurite_02077m_real", expected_device="Acurite", expected_category="Weather Sensor", frequency=433_920_000, short_pulse_us=220, long_pulse_us=440, n_bits=64, preamble_pulses=4, ), TestCase( name="nexus_th_real", expected_device="Nexus", expected_category="Weather Sensor", frequency=433_920_000, short_pulse_us=500, long_pulse_us=1000, n_bits=36, preamble_pulses=8, ), # ── Garage door openers ──────────────────────────────────────────────── TestCase( name="liftmaster_433_real", expected_device="LiftMaster", expected_category="Garage Door Opener", frequency=433_920_000, short_pulse_us=350, long_pulse_us=1050, n_bits=40, sync_gap_us=9000, ), TestCase( name="liftmaster_security2_raw_real", expected_device="LiftMaster Security+ 2.0", expected_category="Garage Door Opener", frequency=390_000_000, short_pulse_us=300, long_pulse_us=600, n_bits=66, sync_gap_us=5000, ), TestCase( name="marantec_real", expected_device="Marantec", expected_category="Garage Door Opener", frequency=433_920_000, short_pulse_us=1000, long_pulse_us=2000, n_bits=12, repeats=5, ), # ── Doorbells ────────────────────────────────────────────────────────── TestCase( name="ge_doorbell_real", expected_device="GE Doorbell", expected_category="Doorbell", frequency=433_920_000, short_pulse_us=250, long_pulse_us=500, n_bits=24, sync_gap_us=5000, ), TestCase( name="byron_doorbell_real", expected_device="Byron Doorbell", expected_category="Doorbell", frequency=433_920_000, short_pulse_us=350, long_pulse_us=1050, n_bits=24, sync_gap_us=10500, ), # ── Fan / LED remotes ────────────────────────────────────────────────── TestCase( name="rgb_led_remote_real", expected_device="RGB LED Remote", expected_category="Remote Control", frequency=433_920_000, short_pulse_us=300, long_pulse_us=900, n_bits=24, sync_gap_us=9000, ), TestCase( name="ceiling_fan_real", expected_device="Hampton Bay", expected_category="Fan Controller", frequency=433_920_000, short_pulse_us=320, long_pulse_us=960, n_bits=12, sync_gap_us=9600, ), TestCase( name="ceiling_fan2_real", expected_device="Harbor Breeze", expected_category="Fan Controller", frequency=433_920_000, short_pulse_us=300, long_pulse_us=900, n_bits=24, sync_gap_us=9000, ), TestCase( name="ceiling_fan_light_real", expected_device="Generic Ceiling Fan", expected_category="Fan Controller", frequency=433_920_000, short_pulse_us=300, long_pulse_us=900, n_bits=24, sync_gap_us=9000, ), ] def generate_sub_file(case: TestCase, path: str) -> None: """Generate a synthetic .sub file matching the test case timing.""" lines = [ "Filetype: Flipper SubGhz RAW File", "Version: 1", f"Frequency: {case.frequency}", "Preset: FuriHalSubGhzPresetOok270Async", "Protocol: RAW", ] # Build RAW_Data pulse train pulses = [] for rep in range(case.repeats): # Optional sync gap (long negative pulse before data) if case.sync_gap_us > 0: pulses.append(case.short_pulse_us) # brief HIGH before sync pulses.append(-case.sync_gap_us) # long LOW sync gap # Optional alternating preamble for _ in range(case.preamble_pulses): pulses.append(case.short_pulse_us) pulses.append(-case.short_pulse_us) # Data bits: alternating SHORT/LONG to simulate mixed data # Use a pseudo-random but reproducible pattern import hashlib seed = int(hashlib.md5(case.name.encode()).hexdigest()[:8], 16) for bit_i in range(case.n_bits): bit = (seed >> (bit_i % 32)) & 1 if bit == 0: pulses.append(case.short_pulse_us) pulses.append(-case.short_pulse_us) else: pulses.append(case.long_pulse_us) pulses.append(-case.short_pulse_us) # Inter-repetition gap if rep < case.repeats - 1: pulses.append(-10000) raw_data = " ".join(str(p) for p in pulses) lines.append(f"RAW_Data: {raw_data}") with open(path, "w") as f: f.write("\n".join(lines) + "\n") def match_name(result_name: str, expected: str) -> bool: """Case-insensitive partial match.""" return expected.lower() in result_name.lower() def run_benchmark() -> None: decoder = get_pattern_decoder() results = { "total": 0, "top1_correct": 0, "top1_category_correct": 0, "top3_correct": 0, "top3_category_correct": 0, "no_match": 0, "details": [], } print("=" * 70) print("Phase 0 Benchmark — Category-Routed Matcher") print("=" * 70) print(f"{'Test':<35} {'Expect':<20} {'Got (top1)':<28} {'Top3?'}") print("-" * 70) with tempfile.TemporaryDirectory() as tmpdir: for case in REAL_WORLD_TEST_CASES: sub_path = os.path.join(tmpdir, f"{case.name}.sub") generate_sub_file(case, sub_path) t0 = time.time() try: metadata = parse_sub_file(sub_path) matches = decoder.decode(metadata) except Exception as e: print(f" ERROR: {case.name}: {e}") continue elapsed_ms = (time.time() - t0) * 1000 results["total"] += 1 if not matches: results["no_match"] += 1 top1_str = "NO MATCH" top1_conf = 0.0 top3_str = "✗" else: top1 = matches[0] top1_str = f"{top1.name[:26]} ({top1.confidence:.0%})" top1_conf = top1.confidence # Check top-1 exact (name match) if match_name(top1.name, case.expected_device): results["top1_correct"] += 1 # Check top-1 category match if top1.category == case.expected_category: results["top1_category_correct"] += 1 # Check top-3 top3_names = [m.name for m in matches[:3]] top3_cats = [m.category for m in matches[:3]] top3_name_hit = any(match_name(n, case.expected_device) for n in top3_names) top3_cat_hit = case.expected_category in top3_cats if top3_name_hit: results["top3_correct"] += 1 top3_str = "✓ (name)" elif top3_cat_hit: results["top3_category_correct"] += 1 top3_str = "~ (cat)" else: top3_str = "✗" detail = { "name": case.name, "expected_device": case.expected_device, "expected_category": case.expected_category, "top1": top1_str, "top3": top3_str, "elapsed_ms": f"{elapsed_ms:.1f}", } results["details"].append(detail) print(f" {case.name:<33} {case.expected_device:<20} {top1_str:<28} {top3_str}") total = results["total"] top1_name = results["top1_correct"] top1_cat = results["top1_category_correct"] top3_name = results["top3_correct"] top3_cat = results["top3_category_correct"] no_match = results["no_match"] top3_any = top3_name + top3_cat print("=" * 70) print(f"\nResults (n={total})") print(f" Top-1 exact match: {top1_name}/{total} = {top1_name/total:.0%}") print(f" Top-1 category match: {top1_cat}/{total} = {top1_cat/total:.0%}") print(f" Top-3 exact match: {top3_name}/{total} = {top3_name/total:.0%}") print(f" Top-3 category match: {top3_any}/{total} = {top3_any/total:.0%} ← primary metric") print(f" No match returned: {no_match}/{total}") print() print(f" Baseline (before Phase 0): top-3 = 0%") print(f" Target: top-3 ≥ 30%") passed = top3_any / total >= 0.30 print(f" Gate: {'✅ PASSED' if passed else '❌ FAILED'}") if __name__ == "__main__": run_benchmark()