leetcrypt
67c376e92c
Trains sklearn RandomForest + GradientBoosting on 14 timing/statistical features from RAW-only UberGuidoZ captures, with a group-aware split (GroupShuffleSplit by device sub-folder) so near-duplicate captures never leak across train/test. The heuristic CategoryRouter is scored on the exact same held-out files for a fair comparison. Best model (gradient_boost): 86.8% top-1 accuracy, 62.5% balanced accuracy vs heuristic 30.0% top-1 / 55.9% routed. Raw accuracy is inflated by the 70%-Garage RAW class balance; balanced accuracy (~2x the heuristic) is the honest number and meets the Phase 3A 60-75% target. Not yet wired into the decoder ensemble. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
47 lines
1002 B
JSON
47 lines
1002 B
JSON
{
|
|
"best_model": "gradient_boost",
|
|
"n_samples": 803,
|
|
"n_train": 583,
|
|
"n_test": 220,
|
|
"features": [
|
|
"freq_mhz",
|
|
"short_pulse_us",
|
|
"long_pulse_us",
|
|
"pulse_ratio",
|
|
"short_gap_us",
|
|
"long_gap_us",
|
|
"gap_ratio",
|
|
"duty_cycle",
|
|
"pulse_count",
|
|
"pulse_mean_abs",
|
|
"pulse_std_abs",
|
|
"pulse_min_abs",
|
|
"pulse_max_abs",
|
|
"preamble_type"
|
|
],
|
|
"class_balance": {
|
|
"Doorbell": 39,
|
|
"Garage Door Opener": 565,
|
|
"Weather Sensor": 12,
|
|
"Fan Controller": 91,
|
|
"Security Sensor": 2,
|
|
"Remote Control": 94
|
|
},
|
|
"metrics": {
|
|
"random_forest": {
|
|
"accuracy": 0.8272727272727273,
|
|
"balanced_accuracy": 0.5789748226457088,
|
|
"macro_f1": 0.5360433604336042
|
|
},
|
|
"gradient_boost": {
|
|
"accuracy": 0.8681818181818182,
|
|
"balanced_accuracy": 0.6253728690437551,
|
|
"macro_f1": 0.5340975664713835
|
|
}
|
|
},
|
|
"heuristic_baseline_same_test": {
|
|
"top1": 0.3,
|
|
"routed": 0.5590909090909091,
|
|
"n_test": 220
|
|
}
|
|
} |