giglez/scripts/download_rf_test_datasets.sh

#!/bin/bash
#
# Download RF Test Datasets for RTL_433 Testing
#
# This script downloads labeled RF signal databases from GitHub
# for testing our RTL_433 integration implementation.
#

set -e

BASE_DIR="/path/to/giglez"
DATA_DIR="$BASE_DIR/data/rf_test_datasets"

mkdir -p "$DATA_DIR"
cd "$DATA_DIR"

echo "========================================================================"
echo "Downloading RF Signal Databases for RTL_433 Testing"
echo "========================================================================"
echo ""

# =============================================================================
# Dataset 1: RTL_433 Official Test Files (HIGHEST QUALITY - 100% labeled)
# =============================================================================

echo "[1/6] Downloading RTL_433 Official Test Files..."
echo "  Repository: merbanan/rtl_433_tests"
echo "  Quality: ★★★★★ (100% labeled, known protocols)"
echo "  Format: .cu8, .json (need conversion)"
echo ""

if [ ! -d "rtl_433_tests" ]; then
    git clone --depth 1 https://github.com/merbanan/rtl_433_tests.git
    echo "  ✓ Downloaded"
else
    echo "  ⏩ Already exists"
fi

# Count test files
CU8_COUNT=$(find rtl_433_tests/tests -name "*.cu8" 2>/dev/null | wc -l)
JSON_COUNT=$(find rtl_433_tests/tests -name "*.json" 2>/dev/null | wc -l)
echo "  → Found $CU8_COUNT .cu8 files, $JSON_COUNT .json files"
echo ""

# =============================================================================
# Dataset 2: Large Flipper Zero .sub Collection (13,717 files)
# =============================================================================

echo "[2/6] Downloading FlipperZero-Subghz-DB (13,717 .sub files)..."
echo "  Repository: Zero-Sploit/FlipperZero-Subghz-DB"
echo "  Quality: ★★★☆☆ (mixed protocols, mostly garage/gate openers)"
echo "  Format: .sub (Flipper format)"
echo ""

if [ ! -d "FlipperZero-Subghz-DB" ]; then
    git clone --depth 1 https://github.com/Zero-Sploit/FlipperZero-Subghz-DB.git
    echo "  ✓ Downloaded"
else
    echo "  ⏩ Already exists"
fi

SUB_COUNT=$(find FlipperZero-Subghz-DB -name "*.sub" 2>/dev/null | wc -l)
echo "  → Found $SUB_COUNT .sub files"
echo ""

# =============================================================================
# Dataset 3: UberGuidoZ Flipper Database (comprehensive collection)
# =============================================================================

echo "[3/6] Downloading UberGuidoZ Flipper Database..."
echo "  Repository: UberGuidoZ/Flipper"
echo "  Quality: ★★★★☆ (well-organized, active community)"
echo "  Format: .sub, .ir, .nfc (multi-format)"
echo ""

if [ ! -d "UberGuidoZ_Flipper" ]; then
    echo "  Note: This is a large repo (~500MB), downloading..."
    git clone --depth 1 https://github.com/UberGuidoZ/Flipper.git UberGuidoZ_Flipper
    echo "  ✓ Downloaded"
else
    echo "  ⏩ Already exists"
fi

UBER_SUB=$(find UberGuidoZ_Flipper -name "*.sub" 2>/dev/null | wc -l)
echo "  → Found $UBER_SUB .sub files"
echo ""

# =============================================================================
# Dataset 4: Weather Station Spoofer (Generated .sub files)
# =============================================================================

echo "[4/6] Downloading WSSFlipperZero (Weather Station Spoofer)..."
echo "  Repository: timdigga/WSSFlipperZero"
echo "  Quality: ★★★★☆ (synthetic weather sensor signals)"
echo "  Format: .sub (generated, not captured)"
echo ""

if [ ! -d "WSSFlipperZero" ]; then
    git clone --depth 1 https://github.com/timdigga/WSSFlipperZero.git
    echo "  ✓ Downloaded"
else
    echo "  ⏩ Already exists"
fi

echo "  → Contains weather sensor .sub generators"
echo ""

# =============================================================================
# Dataset 5: Full Flipper Database (comprehensive collection)
# =============================================================================

echo "[5/6] Downloading Full Flipper Database..."
echo "  Repository: DefinetlyNotAI/Full_Flipper_Database"
echo "  Quality: ★★★☆☆ (large collection, mixed quality)"
echo "  Format: .sub, .ir, .nfc, etc"
echo ""

if [ ! -d "Full_Flipper_Database" ]; then
    git clone --depth 1 https://github.com/DefinetlyNotAI/Full_Flipper_Database.git
    echo "  ✓ Downloaded"
else
    echo "  ⏩ Already exists"
fi

FULL_SUB=$(find Full_Flipper_Database -name "*.sub" 2>/dev/null | wc -l)
echo "  → Found $FULL_SUB .sub files"
echo ""

# =============================================================================
# Dataset 6: Awesome Flipper Zero (curated links)
# =============================================================================

echo "[6/6] Downloading Awesome Flipper Zero (resource list)..."
echo "  Repository: djsime1/awesome-flipperzero"
echo "  Quality: ★★★★★ (curated list of resources)"
echo "  Format: README with links"
echo ""

if [ ! -d "awesome-flipperzero" ]; then
    git clone --depth 1 https://github.com/djsime1/awesome-flipperzero.git
    echo "  ✓ Downloaded"
else
    echo "  ⏩ Already exists"
fi

echo "  → Contains curated links to more datasets"
echo ""

# =============================================================================
# Analysis and Filtering
# =============================================================================

echo "========================================================================"
echo "Dataset Analysis"
echo "========================================================================"
echo ""

echo "Searching for weather sensor captures..."
echo ""

# Search for weather-related .sub files
find . -name "*.sub" 2>/dev/null | \
    xargs grep -l "Oregon\|Acurite\|LaCrosse\|Weather\|Temperature" 2>/dev/null | \
    head -20 > weather_sensors.txt || true

WEATHER_COUNT=$(cat weather_sensors.txt | wc -l)
echo "  → Found $WEATHER_COUNT potential weather sensor captures"

if [ $WEATHER_COUNT -gt 0 ]; then
    echo ""
    echo "Top 10 weather sensor files:"
    head -10 weather_sensors.txt | while read file; do
        echo "  - $file"
    done
fi

echo ""

# =============================================================================
# Create filtered test dataset
# =============================================================================

echo "Creating filtered test dataset..."
TEST_DIR="$BASE_DIR/data/test_rtl433_real"
mkdir -p "$TEST_DIR"

# Copy weather sensor files
if [ $WEATHER_COUNT -gt 0 ]; then
    head -10 weather_sensors.txt | while read file; do
        basename=$(basename "$file")
        cp "$file" "$TEST_DIR/$basename" 2>/dev/null || true
    done

    COPIED=$(ls "$TEST_DIR"/*.sub 2>/dev/null | wc -l)
    echo "  ✓ Copied $COPIED weather sensor files to $TEST_DIR"
fi

echo ""

# =============================================================================
# Summary
# =============================================================================

echo "========================================================================"
echo "Download Summary"
echo "========================================================================"
echo ""
echo "Total datasets downloaded: 6"
echo ""
echo "By Format:"
echo "  - .cu8 (RTL_433):      $CU8_COUNT files"
echo "  - .sub (Flipper):      $(find . -name "*.sub" 2>/dev/null | wc -l) files"
echo "  - .json (metadata):    $JSON_COUNT files"
echo ""
echo "By Category:"
echo "  - Weather sensors:     $WEATHER_COUNT files"
echo "  - Total captures:      $(find . -name "*.sub" -o -name "*.cu8" 2>/dev/null | wc -l) files"
echo ""
echo "Next Steps:"
echo "  1. Run RTL_433 tests with real data:"
echo "     PYTHONPATH=$BASE_DIR python3 scripts/test_rtl433_with_known_devices.py"
echo ""
echo "  2. Convert RTL_433 .cu8 files to .sub format (requires converter)"
echo ""
echo "  3. Check weather_sensors.txt for high-quality captures"
echo ""
echo "========================================================================"