leetcrypt
9f73595b20
- Expanded protocol database from 18 → 299 signatures (16.6x increase) - Imported 281 protocols from RTL_433 open-source database (286 total devices) - Created automated import script: scripts/import_rtl433_protocols.py - Generated rtl433_protocols_imported.py with timing/frequency/modulation data - Updated protocol_database.py to include RTL433_PROTOCOLS - All 26 tests passing Breakdown by category: - Weather: 116 protocols - Sensors: 36 protocols - TPMS: 25 protocols - Security: 23 protocols - Home Automation: 18 protocols - Other: 50+ protocols Frequency coverage: - 433.92 MHz: 248 protocols - 315.00 MHz: 32 protocols - 915.00 MHz: 1 protocol This provides comprehensive coverage of Sub-GHz IoT devices for accurate identification from raw RF captures.
326 lines
12 KiB
Python
Executable File
326 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Upload Log Analyzer
|
|
|
|
Analyzes upload logs from beta testing to identify patterns, common errors,
|
|
and performance bottlenecks.
|
|
|
|
Usage:
|
|
python3 scripts/analyze_upload_logs.py --date 2026-01-15
|
|
python3 scripts/analyze_upload_logs.py --last 7 # Last 7 days
|
|
python3 scripts/analyze_upload_logs.py --upload-id upload_20260115_123456
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from datetime import datetime, timedelta
|
|
from collections import Counter, defaultdict
|
|
from typing import List, Dict, Any
|
|
import statistics
|
|
|
|
|
|
class UploadLogAnalyzer:
|
|
"""Analyze upload logs and generate reports"""
|
|
|
|
def __init__(self, log_dir: str = "logs/uploads"):
|
|
self.log_dir = Path(log_dir)
|
|
|
|
def analyze_date(self, date: str) -> Dict[str, Any]:
|
|
"""Analyze logs for a specific date (YYYY-MM-DD)"""
|
|
|
|
# Read main upload log
|
|
log_file = self.log_dir / f"uploads_{date}.log"
|
|
error_log = self.log_dir / f"upload_errors_{date}.log"
|
|
structured_log = self.log_dir / f"uploads_structured_{date}.jsonl"
|
|
|
|
if not log_file.exists():
|
|
return {"error": f"Log file not found: {log_file}"}
|
|
|
|
stats = {
|
|
"date": date,
|
|
"total_uploads": 0,
|
|
"total_files": 0,
|
|
"successful_files": 0,
|
|
"failed_files": 0,
|
|
"duplicate_files": 0,
|
|
"errors_by_type": Counter(),
|
|
"errors_by_stage": Counter(),
|
|
"parse_errors": [],
|
|
"gps_errors": [],
|
|
"performance": {
|
|
"parse_times": [],
|
|
"matching_times": [],
|
|
"total_times": []
|
|
},
|
|
"top_failures": [],
|
|
"user_stats": defaultdict(int)
|
|
}
|
|
|
|
# Parse structured JSON log
|
|
if structured_log.exists():
|
|
with open(structured_log, 'r') as f:
|
|
for line in f:
|
|
try:
|
|
entry = json.loads(line)
|
|
self._process_log_entry(entry, stats)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
# Parse error log for details
|
|
if error_log.exists():
|
|
with open(error_log, 'r') as f:
|
|
for line in f:
|
|
self._process_error_line(line, stats)
|
|
|
|
# Calculate summary statistics
|
|
self._calculate_summary(stats)
|
|
|
|
return stats
|
|
|
|
def _process_log_entry(self, entry: Dict[str, Any], stats: Dict[str, Any]):
|
|
"""Process a single structured log entry"""
|
|
event = entry.get("event")
|
|
|
|
if event == "upload_attempt":
|
|
stats["total_uploads"] += 1
|
|
stats["total_files"] += entry.get("file_count", 0)
|
|
if entry.get("user_id"):
|
|
stats["user_stats"]["authenticated"] += 1
|
|
else:
|
|
stats["user_stats"]["anonymous"] += 1
|
|
|
|
elif event == "file_processing":
|
|
stage = entry.get("stage")
|
|
success = entry.get("success")
|
|
|
|
if stage == "complete" and success:
|
|
stats["successful_files"] += 1
|
|
elif not success:
|
|
if stage == "validation" and entry.get("metadata", {}).get("status") == "duplicate":
|
|
stats["duplicate_files"] += 1
|
|
else:
|
|
stats["failed_files"] += 1
|
|
stats["errors_by_stage"][stage] += 1
|
|
|
|
# Track performance
|
|
duration = entry.get("duration_ms")
|
|
if duration and success:
|
|
if stage == "parsing":
|
|
stats["performance"]["parse_times"].append(duration)
|
|
elif stage == "complete":
|
|
stats["performance"]["total_times"].append(duration)
|
|
|
|
elif event == "parse_error":
|
|
stats["parse_errors"].append({
|
|
"filename": entry.get("filename"),
|
|
"error_type": entry.get("error_type"),
|
|
"error_message": entry.get("error_message"),
|
|
"upload_id": entry.get("upload_id")
|
|
})
|
|
stats["errors_by_type"][entry.get("error_type", "Unknown")] += 1
|
|
|
|
elif event == "gps_validation_error":
|
|
stats["gps_errors"].append({
|
|
"filename": entry.get("filename"),
|
|
"latitude": entry.get("latitude"),
|
|
"longitude": entry.get("longitude"),
|
|
"reason": entry.get("reason")
|
|
})
|
|
stats["errors_by_type"]["GPS Validation"] += 1
|
|
|
|
elif event == "matching_results":
|
|
duration = entry.get("duration_ms")
|
|
if duration:
|
|
stats["performance"]["matching_times"].append(duration)
|
|
|
|
elif event == "performance_metrics":
|
|
# Aggregate performance metrics
|
|
for key, value in entry.items():
|
|
if key.endswith("_ms") and isinstance(value, (int, float)):
|
|
perf_key = key.replace("_ms", "_times")
|
|
if perf_key not in stats["performance"]:
|
|
stats["performance"][perf_key] = []
|
|
stats["performance"][perf_key].append(value)
|
|
|
|
def _process_error_line(self, line: str, stats: Dict[str, Any]):
|
|
"""Process error log line"""
|
|
# Extract error type from log line
|
|
error_match = re.search(r'(Exception|Error): (.+)', line)
|
|
if error_match:
|
|
error_type = error_match.group(1)
|
|
stats["errors_by_type"][error_type] += 1
|
|
|
|
def _calculate_summary(self, stats: Dict[str, Any]):
|
|
"""Calculate summary statistics"""
|
|
# Success rate
|
|
total = stats["successful_files"] + stats["failed_files"]
|
|
if total > 0:
|
|
stats["success_rate"] = round(stats["successful_files"] / total * 100, 1)
|
|
else:
|
|
stats["success_rate"] = 0.0
|
|
|
|
# Performance statistics
|
|
for key, times in stats["performance"].items():
|
|
if times:
|
|
stats["performance"][f"{key}_avg"] = round(statistics.mean(times), 2)
|
|
stats["performance"][f"{key}_median"] = round(statistics.median(times), 2)
|
|
stats["performance"][f"{key}_p95"] = round(statistics.quantiles(times, n=20)[18], 2) if len(times) > 20 else None
|
|
|
|
# Top failure reasons
|
|
stats["top_failures"] = stats["errors_by_type"].most_common(10)
|
|
|
|
def print_report(self, stats: Dict[str, Any]):
|
|
"""Print formatted report"""
|
|
print("=" * 80)
|
|
print(f"Upload Log Analysis - {stats['date']}")
|
|
print("=" * 80)
|
|
|
|
# Overview
|
|
print("\n📊 OVERVIEW")
|
|
print(f" Total Uploads: {stats['total_uploads']}")
|
|
print(f" Total Files: {stats['total_files']}")
|
|
print(f" Successful: {stats['successful_files']} ({stats['success_rate']}%)")
|
|
print(f" Failed: {stats['failed_files']}")
|
|
print(f" Duplicates: {stats['duplicate_files']}")
|
|
|
|
# User statistics
|
|
print("\n👥 USERS")
|
|
print(f" Authenticated: {stats['user_stats']['authenticated']}")
|
|
print(f" Anonymous: {stats['user_stats']['anonymous']}")
|
|
|
|
# Performance
|
|
print("\n⚡ PERFORMANCE")
|
|
perf = stats["performance"]
|
|
if perf.get("parse_times_avg"):
|
|
print(f" Parse Time: {perf['parse_times_avg']:.0f}ms avg, {perf['parse_times_median']:.0f}ms median")
|
|
if perf.get("matching_times_avg"):
|
|
print(f" Matching Time: {perf['matching_times_avg']:.0f}ms avg, {perf['matching_times_median']:.0f}ms median")
|
|
if perf.get("total_times_avg"):
|
|
print(f" Total Time: {perf['total_times_avg']:.0f}ms avg, {perf['total_times_median']:.0f}ms median")
|
|
|
|
# Errors
|
|
if stats["top_failures"]:
|
|
print("\n❌ TOP ERRORS")
|
|
for error_type, count in stats["top_failures"]:
|
|
print(f" {error_type:30} {count:5} occurrences")
|
|
|
|
# Parse errors detail
|
|
if stats["parse_errors"]:
|
|
print(f"\n🔍 PARSE ERRORS ({len(stats['parse_errors'])})")
|
|
for error in stats["parse_errors"][:5]: # Show first 5
|
|
print(f" {error['filename']:30} {error['error_type']}: {error['error_message'][:50]}")
|
|
|
|
# GPS errors detail
|
|
if stats["gps_errors"]:
|
|
print(f"\n📍 GPS ERRORS ({len(stats['gps_errors'])})")
|
|
for error in stats["gps_errors"][:5]: # Show first 5
|
|
print(f" {error['filename']:30} ({error['latitude']}, {error['longitude']}) - {error['reason']}")
|
|
|
|
print("\n" + "=" * 80)
|
|
|
|
def analyze_upload_id(self, upload_id: str) -> Dict[str, Any]:
|
|
"""Analyze a specific upload"""
|
|
# Search all structured logs
|
|
results = {
|
|
"upload_id": upload_id,
|
|
"events": [],
|
|
"files": [],
|
|
"errors": [],
|
|
"timeline": []
|
|
}
|
|
|
|
for log_file in self.log_dir.glob("uploads_structured_*.jsonl"):
|
|
with open(log_file, 'r') as f:
|
|
for line in f:
|
|
try:
|
|
entry = json.loads(line)
|
|
if entry.get("upload_id") == upload_id:
|
|
results["events"].append(entry)
|
|
results["timeline"].append({
|
|
"timestamp": entry.get("timestamp"),
|
|
"event": entry.get("event"),
|
|
"details": entry
|
|
})
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
# Sort timeline
|
|
results["timeline"].sort(key=lambda x: x["timestamp"])
|
|
|
|
return results
|
|
|
|
def print_upload_details(self, details: Dict[str, Any]):
|
|
"""Print detailed upload report"""
|
|
print("=" * 80)
|
|
print(f"Upload Details: {details['upload_id']}")
|
|
print("=" * 80)
|
|
|
|
print(f"\nTotal Events: {len(details['events'])}")
|
|
|
|
print("\n📅 TIMELINE:")
|
|
for event in details["timeline"]:
|
|
timestamp = event["timestamp"].split('T')[1][:12] # HH:MM:SS.mmm
|
|
print(f" {timestamp} | {event['event']:20} | {self._format_event_details(event['details'])}")
|
|
|
|
def _format_event_details(self, event: Dict[str, Any]) -> str:
|
|
"""Format event details for display"""
|
|
event_type = event.get("event")
|
|
|
|
if event_type == "upload_attempt":
|
|
return f"{event.get('file_count')} files, {event.get('total_size_mb')} MB"
|
|
elif event_type == "file_processing":
|
|
success = "✓" if event.get("success") else "✗"
|
|
return f"{success} {event.get('filename')} - {event.get('stage')} ({event.get('duration_ms', 0):.0f}ms)"
|
|
elif event_type == "parse_error":
|
|
return f"❌ {event.get('filename')} - {event.get('error_type')}"
|
|
elif event_type == "matching_results":
|
|
return f"🔍 {event.get('match_count')} matches ({event.get('duration_ms', 0):.0f}ms)"
|
|
elif event_type == "upload_complete":
|
|
return f"✅ {event.get('successful_count')} success, {event.get('failed_count')} failed"
|
|
else:
|
|
return json.dumps(event, indent=2)[:100]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Analyze upload logs from beta testing")
|
|
parser.add_argument("--date", help="Date to analyze (YYYY-MM-DD)")
|
|
parser.add_argument("--last", type=int, help="Analyze last N days")
|
|
parser.add_argument("--upload-id", help="Analyze specific upload ID")
|
|
parser.add_argument("--log-dir", default="logs/uploads", help="Log directory")
|
|
|
|
args = parser.parse_args()
|
|
|
|
analyzer = UploadLogAnalyzer(log_dir=args.log_dir)
|
|
|
|
if args.upload_id:
|
|
# Analyze specific upload
|
|
details = analyzer.analyze_upload_id(args.upload_id)
|
|
analyzer.print_upload_details(details)
|
|
|
|
elif args.date:
|
|
# Analyze specific date
|
|
stats = analyzer.analyze_date(args.date)
|
|
analyzer.print_report(stats)
|
|
|
|
elif args.last:
|
|
# Analyze last N days
|
|
for i in range(args.last):
|
|
date = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
|
|
print(f"\n{'='*80}")
|
|
print(f"Analyzing {date}")
|
|
print('='*80)
|
|
stats = analyzer.analyze_date(date)
|
|
analyzer.print_report(stats)
|
|
|
|
else:
|
|
# Default: analyze today
|
|
today = datetime.now().strftime("%Y-%m-%d")
|
|
stats = analyzer.analyze_date(today)
|
|
analyzer.print_report(stats)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|