feat: RTL_433 protocol database import - iteration 1/5
- Expanded protocol database from 18 → 299 signatures (16.6x increase) - Imported 281 protocols from RTL_433 open-source database (286 total devices) - Created automated import script: scripts/import_rtl433_protocols.py - Generated rtl433_protocols_imported.py with timing/frequency/modulation data - Updated protocol_database.py to include RTL433_PROTOCOLS - All 26 tests passing Breakdown by category: - Weather: 116 protocols - Sensors: 36 protocols - TPMS: 25 protocols - Security: 23 protocols - Home Automation: 18 protocols - Other: 50+ protocols Frequency coverage: - 433.92 MHz: 248 protocols - 315.00 MHz: 32 protocols - 915.00 MHz: 1 protocol This provides comprehensive coverage of Sub-GHz IoT devices for accurate identification from raw RF captures.
This commit is contained in:
leetcrypt
@@ -0,0 +1,380 @@
|
||||
/**
|
||||
* Flipper Zero .sub File Parser (JavaScript)
|
||||
*
|
||||
* Parses .sub files in the browser for real-time device identification
|
||||
* Port of src/parser/sub_parser.py
|
||||
*/
|
||||
|
||||
// Modulation types
|
||||
const Modulation = {
|
||||
OOK: 'OOK',
|
||||
FSK2: '2FSK',
|
||||
FSK4: '4FSK',
|
||||
ASK: 'ASK',
|
||||
UNKNOWN: 'UNKNOWN'
|
||||
};
|
||||
|
||||
// Preset mapping (Flipper firmware presets)
|
||||
const PRESET_TO_MODULATION = {
|
||||
'FuriHalSubGhzPresetOok270Async': Modulation.OOK,
|
||||
'FuriHalSubGhzPresetOok650Async': Modulation.OOK,
|
||||
'FuriHalSubGhzPreset2FSKDev238Async': Modulation.FSK2,
|
||||
'FuriHalSubGhzPreset2FSKDev476Async': Modulation.FSK2,
|
||||
'FuriHalSubGhzPresetMSK99_97KbAsync': Modulation.FSK2,
|
||||
'FuriHalSubGhzPresetGFSK9_99KbAsync': Modulation.FSK2,
|
||||
};
|
||||
|
||||
/**
|
||||
* Signal Metadata Structure
|
||||
*/
|
||||
class SignalMetadata {
|
||||
constructor() {
|
||||
this.filetype = null;
|
||||
this.version = null;
|
||||
this.frequency = null; // Hz
|
||||
this.preset = null;
|
||||
this.protocol = null;
|
||||
this.modulation = null;
|
||||
this.bit_length = null;
|
||||
this.key_data = null; // Hex string
|
||||
this.timing_element = null; // Microseconds
|
||||
this.raw_data = null; // Array of integers [pulse, -gap, pulse, -gap, ...]
|
||||
this.raw_format = null; // 'RAW', 'BinRAW', or 'KEY'
|
||||
|
||||
// Computed statistics (for RAW files)
|
||||
this.pulse_count = null;
|
||||
this.average_pulse_width = null;
|
||||
this.total_duration_ms = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute statistics from raw pulse data
|
||||
*/
|
||||
computeStatistics() {
|
||||
if (!this.raw_data || this.raw_data.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pulses = this.raw_data.filter(x => x > 0);
|
||||
const gaps = this.raw_data.filter(x => x < 0).map(x => Math.abs(x));
|
||||
|
||||
this.pulse_count = pulses.length;
|
||||
|
||||
if (pulses.length > 0) {
|
||||
this.average_pulse_width = pulses.reduce((a, b) => a + b, 0) / pulses.length;
|
||||
}
|
||||
|
||||
// Total duration in milliseconds
|
||||
const total_us = this.raw_data.map(Math.abs).reduce((a, b) => a + b, 0);
|
||||
this.total_duration_ms = total_us / 1000;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a .sub file from text content
|
||||
*
|
||||
* @param {string} content - The text content of the .sub file
|
||||
* @returns {SignalMetadata} Parsed metadata
|
||||
*/
|
||||
function parseSubFile(content) {
|
||||
const metadata = new SignalMetadata();
|
||||
const lines = content.split('\n').map(line => line.trim()).filter(line => line.length > 0);
|
||||
|
||||
for (const line of lines) {
|
||||
const colonIndex = line.indexOf(':');
|
||||
if (colonIndex === -1) continue;
|
||||
|
||||
const key = line.substring(0, colonIndex).trim();
|
||||
const value = line.substring(colonIndex + 1).trim();
|
||||
|
||||
// Parse fields
|
||||
switch (key) {
|
||||
case 'Filetype':
|
||||
metadata.filetype = value;
|
||||
break;
|
||||
|
||||
case 'Version':
|
||||
metadata.version = parseInt(value);
|
||||
break;
|
||||
|
||||
case 'Frequency':
|
||||
metadata.frequency = parseInt(value);
|
||||
|
||||
// Validate frequency range (300 MHz - 928 MHz)
|
||||
if (metadata.frequency < 300000000 || metadata.frequency > 928000000) {
|
||||
console.warn(`Frequency ${metadata.frequency} Hz outside Sub-GHz range`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'Preset':
|
||||
metadata.preset = value;
|
||||
metadata.modulation = PRESET_TO_MODULATION[value] || Modulation.UNKNOWN;
|
||||
break;
|
||||
|
||||
case 'Protocol':
|
||||
metadata.protocol = value;
|
||||
|
||||
// Determine format type
|
||||
if (value === 'RAW') {
|
||||
metadata.raw_format = 'RAW';
|
||||
} else if (value === 'BinRAW') {
|
||||
metadata.raw_format = 'BinRAW';
|
||||
} else {
|
||||
metadata.raw_format = 'KEY';
|
||||
}
|
||||
break;
|
||||
|
||||
case 'Bit':
|
||||
metadata.bit_length = parseInt(value);
|
||||
break;
|
||||
|
||||
case 'Key':
|
||||
metadata.key_data = value;
|
||||
break;
|
||||
|
||||
case 'TE':
|
||||
metadata.timing_element = parseInt(value);
|
||||
break;
|
||||
|
||||
case 'RAW_Data':
|
||||
// Parse timing array
|
||||
const timings = value.split(/\s+/)
|
||||
.map(x => parseInt(x))
|
||||
.filter(x => !isNaN(x));
|
||||
|
||||
metadata.raw_data = timings;
|
||||
break;
|
||||
|
||||
default:
|
||||
// Ignore unknown fields
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Compute statistics for RAW files
|
||||
if (metadata.raw_data) {
|
||||
metadata.computeStatistics();
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract statistical features from RAW pulse data
|
||||
* (For use with statistical ML classifier)
|
||||
*
|
||||
* @param {SignalMetadata} metadata
|
||||
* @returns {Float32Array} Feature vector (length 47)
|
||||
*/
|
||||
function extractStatisticalFeatures(metadata) {
|
||||
const features = new Float32Array(47);
|
||||
|
||||
if (!metadata.raw_data || metadata.raw_data.length === 0) {
|
||||
return features; // All zeros
|
||||
}
|
||||
|
||||
const pulses = metadata.raw_data.filter(x => x > 0);
|
||||
const gaps = metadata.raw_data.filter(x => x < 0).map(x => Math.abs(x));
|
||||
|
||||
// Helper functions
|
||||
const mean = arr => arr.reduce((a, b) => a + b, 0) / arr.length;
|
||||
const median = arr => {
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
|
||||
};
|
||||
const std = arr => {
|
||||
const m = mean(arr);
|
||||
return Math.sqrt(arr.reduce((sum, x) => sum + (x - m) ** 2, 0) / arr.length);
|
||||
};
|
||||
|
||||
// Timing features (16)
|
||||
let idx = 0;
|
||||
features[idx++] = mean(pulses);
|
||||
features[idx++] = median(pulses);
|
||||
features[idx++] = std(pulses);
|
||||
features[idx++] = Math.min(...pulses);
|
||||
features[idx++] = Math.max(...pulses);
|
||||
|
||||
features[idx++] = mean(gaps);
|
||||
features[idx++] = median(gaps);
|
||||
features[idx++] = std(gaps);
|
||||
features[idx++] = Math.min(...gaps);
|
||||
features[idx++] = Math.max(...gaps);
|
||||
|
||||
features[idx++] = mean(pulses) / mean(gaps); // Pulse/gap ratio
|
||||
features[idx++] = mean(pulses) / (mean(pulses) + mean(gaps)); // Duty cycle
|
||||
|
||||
// K-means-like clustering for short/long pulses (simplified)
|
||||
const pulsesSorted = [...pulses].sort((a, b) => a - b);
|
||||
const shortPulse = pulsesSorted[Math.floor(pulsesSorted.length * 0.25)];
|
||||
const longPulse = pulsesSorted[Math.floor(pulsesSorted.length * 0.75)];
|
||||
features[idx++] = shortPulse;
|
||||
features[idx++] = longPulse;
|
||||
|
||||
features[idx++] = std(pulses) / mean(pulses); // Coefficient of variation (pulses)
|
||||
features[idx++] = std(gaps) / mean(gaps); // Coefficient of variation (gaps)
|
||||
|
||||
// Frequency domain (12) - simplified FFT features
|
||||
// (Full FFT would require library like FFT.js - simplified here)
|
||||
features[idx++] = 1000000 / mean(pulses); // Estimated dominant frequency (Hz)
|
||||
features[idx++] = metadata.total_duration_ms || 0; // Total energy proxy
|
||||
features[idx++] = metadata.pulse_count || 0;
|
||||
|
||||
// Zero-crossing rate
|
||||
let zero_crossings = 0;
|
||||
for (let i = 1; i < metadata.raw_data.length; i++) {
|
||||
if (metadata.raw_data[i] * metadata.raw_data[i-1] < 0) {
|
||||
zero_crossings++;
|
||||
}
|
||||
}
|
||||
features[idx++] = zero_crossings / metadata.raw_data.length;
|
||||
|
||||
// Fill remaining frequency features with zeros (would need FFT)
|
||||
for (let i = 0; i < 8; i++) {
|
||||
features[idx++] = 0;
|
||||
}
|
||||
|
||||
// Pattern features (10)
|
||||
// Entropy of pulse width distribution (simplified)
|
||||
const pulseBins = {};
|
||||
for (const p of pulses) {
|
||||
const bin = Math.floor(p / 100) * 100;
|
||||
pulseBins[bin] = (pulseBins[bin] || 0) + 1;
|
||||
}
|
||||
const probs = Object.values(pulseBins).map(c => c / pulses.length);
|
||||
const entropy = -probs.reduce((sum, p) => sum + (p > 0 ? p * Math.log2(p) : 0), 0);
|
||||
features[idx++] = entropy;
|
||||
|
||||
// Peak/valley counts (simplified)
|
||||
features[idx++] = pulses.length;
|
||||
features[idx++] = gaps.length;
|
||||
|
||||
// Longest run of similar pulses
|
||||
let longestRun = 1;
|
||||
let currentRun = 1;
|
||||
for (let i = 1; i < pulses.length; i++) {
|
||||
if (Math.abs(pulses[i] - pulses[i-1]) < pulses[i] * 0.2) {
|
||||
currentRun++;
|
||||
} else {
|
||||
longestRun = Math.max(longestRun, currentRun);
|
||||
currentRun = 1;
|
||||
}
|
||||
}
|
||||
features[idx++] = longestRun;
|
||||
|
||||
// Fill remaining pattern features
|
||||
for (let i = 0; i < 6; i++) {
|
||||
features[idx++] = 0;
|
||||
}
|
||||
|
||||
// Metadata features (9)
|
||||
features[idx++] = (metadata.frequency || 433920000) / 1000000; // Frequency in MHz
|
||||
|
||||
// Modulation one-hot encoding (3 features)
|
||||
features[idx++] = metadata.modulation === Modulation.OOK ? 1 : 0;
|
||||
features[idx++] = metadata.modulation === Modulation.FSK2 ? 1 : 0;
|
||||
features[idx++] = metadata.modulation === Modulation.ASK ? 1 : 0;
|
||||
|
||||
features[idx++] = metadata.pulse_count || 0;
|
||||
features[idx++] = metadata.total_duration_ms || 0;
|
||||
|
||||
// Estimated bit rate
|
||||
const bitRate = metadata.total_duration_ms > 0
|
||||
? (metadata.bit_length || 0) / (metadata.total_duration_ms / 1000)
|
||||
: 0;
|
||||
features[idx++] = bitRate;
|
||||
|
||||
// Fill remaining metadata features
|
||||
for (let i = 0; i < 2; i++) {
|
||||
features[idx++] = 0;
|
||||
}
|
||||
|
||||
return features;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize raw pulse data for CNN input
|
||||
* Pads or truncates to fixed length (512 samples) and normalizes to [-1, 1]
|
||||
*
|
||||
* @param {number[]} rawData - Raw pulse array
|
||||
* @param {number} targetLength - Target length (default 512)
|
||||
* @returns {Float32Array} Normalized array
|
||||
*/
|
||||
function normalizeForCNN(rawData, targetLength = 512) {
|
||||
const result = new Float32Array(targetLength);
|
||||
|
||||
if (!rawData || rawData.length === 0) {
|
||||
return result; // All zeros
|
||||
}
|
||||
|
||||
// Find max absolute value for normalization
|
||||
const maxAbs = Math.max(...rawData.map(Math.abs));
|
||||
|
||||
// Pad or truncate
|
||||
for (let i = 0; i < targetLength; i++) {
|
||||
if (i < rawData.length) {
|
||||
result[i] = rawData[i] / maxAbs; // Normalize to [-1, 1]
|
||||
} else {
|
||||
result[i] = 0; // Padding
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a parsed .sub file
|
||||
*
|
||||
* @param {SignalMetadata} metadata
|
||||
* @returns {object} {valid: boolean, errors: string[]}
|
||||
*/
|
||||
function validateSubFile(metadata) {
|
||||
const errors = [];
|
||||
|
||||
if (!metadata.frequency) {
|
||||
errors.push("Missing frequency");
|
||||
} else if (metadata.frequency < 300000000 || metadata.frequency > 928000000) {
|
||||
errors.push(`Frequency ${metadata.frequency / 1e6} MHz outside Sub-GHz range (300-928 MHz)`);
|
||||
}
|
||||
|
||||
if (!metadata.protocol) {
|
||||
errors.push("Missing protocol");
|
||||
}
|
||||
|
||||
if (metadata.raw_format === 'RAW' && (!metadata.raw_data || metadata.raw_data.length === 0)) {
|
||||
errors.push("RAW format but no RAW_Data found");
|
||||
}
|
||||
|
||||
if (metadata.raw_format === 'KEY' && !metadata.key_data) {
|
||||
errors.push("KEY format but no Key field found");
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors
|
||||
};
|
||||
}
|
||||
|
||||
// Export for use in other modules
|
||||
if (typeof module !== 'undefined' && module.exports) {
|
||||
// Node.js
|
||||
module.exports = {
|
||||
parseSubFile,
|
||||
extractStatisticalFeatures,
|
||||
normalizeForCNN,
|
||||
validateSubFile,
|
||||
SignalMetadata,
|
||||
Modulation
|
||||
};
|
||||
} else {
|
||||
// Browser
|
||||
window.SubParser = {
|
||||
parseSubFile,
|
||||
extractStatisticalFeatures,
|
||||
normalizeForCNN,
|
||||
validateSubFile,
|
||||
SignalMetadata,
|
||||
Modulation
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user