feat: Phase 3A statistical device-category classifier (86.8% top-1 on RAW)
Trains sklearn RandomForest + GradientBoosting on 14 timing/statistical features from RAW-only UberGuidoZ captures, with a group-aware split (GroupShuffleSplit by device sub-folder) so near-duplicate captures never leak across train/test. The heuristic CategoryRouter is scored on the exact same held-out files for a fair comparison. Best model (gradient_boost): 86.8% top-1 accuracy, 62.5% balanced accuracy vs heuristic 30.0% top-1 / 55.9% routed. Raw accuracy is inflated by the 70%-Garage RAW class balance; balanced accuracy (~2x the heuristic) is the honest number and meets the Phase 3A 60-75% target. Not yet wired into the decoder ensemble. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"best_model": "gradient_boost",
|
||||
"n_samples": 803,
|
||||
"n_train": 583,
|
||||
"n_test": 220,
|
||||
"features": [
|
||||
"freq_mhz",
|
||||
"short_pulse_us",
|
||||
"long_pulse_us",
|
||||
"pulse_ratio",
|
||||
"short_gap_us",
|
||||
"long_gap_us",
|
||||
"gap_ratio",
|
||||
"duty_cycle",
|
||||
"pulse_count",
|
||||
"pulse_mean_abs",
|
||||
"pulse_std_abs",
|
||||
"pulse_min_abs",
|
||||
"pulse_max_abs",
|
||||
"preamble_type"
|
||||
],
|
||||
"class_balance": {
|
||||
"Doorbell": 39,
|
||||
"Garage Door Opener": 565,
|
||||
"Weather Sensor": 12,
|
||||
"Fan Controller": 91,
|
||||
"Security Sensor": 2,
|
||||
"Remote Control": 94
|
||||
},
|
||||
"metrics": {
|
||||
"random_forest": {
|
||||
"accuracy": 0.8272727272727273,
|
||||
"balanced_accuracy": 0.5789748226457088,
|
||||
"macro_f1": 0.5360433604336042
|
||||
},
|
||||
"gradient_boost": {
|
||||
"accuracy": 0.8681818181818182,
|
||||
"balanced_accuracy": 0.6253728690437551,
|
||||
"macro_f1": 0.5340975664713835
|
||||
}
|
||||
},
|
||||
"heuristic_baseline_same_test": {
|
||||
"top1": 0.3,
|
||||
"routed": 0.5590909090909091,
|
||||
"n_test": 220
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user