bigbrother/scripts/operator/extract_emails.sh

#!/usr/bin/env bash
# BigBrother Operator Script — Extract Emails from PCAPs
#
# Extracts SMTP email messages from network captures. Reconstructs
# complete emails including headers, body, and attachments.
#
# Unencrypted SMTP (port 25/587 without STARTTLS) is still common on
# internal networks, especially between mail servers and printers/scanners.
#
# Usage: ./extract_emails.sh <pcap_dir> [output_dir]
#
# Requires: tshark

set -euo pipefail

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'

# ---------------------------------------------------------------------------
# Arguments
# ---------------------------------------------------------------------------

PCAP_DIR="${1:-}"
OUTPUT_DIR="${2:-$(pwd)/extracted-emails-$(date +%Y%m%d-%H%M%S)}"

if [[ -z "$PCAP_DIR" ]]; then
    echo -e "${RED}Usage: $0 <pcap_dir> [output_dir]${NC}"
    exit 1
fi

if [[ ! -d "$PCAP_DIR" ]]; then
    echo -e "${RED}[-] PCAP directory not found: $PCAP_DIR${NC}"
    exit 1
fi

if ! command -v tshark &>/dev/null; then
    echo -e "${RED}[-] tshark not found — install with: apt install tshark${NC}"
    exit 1
fi

mkdir -p "$OUTPUT_DIR"/{raw,parsed,attachments,summary}

echo -e "${CYAN}[*] BigBrother Email Extraction${NC}"
echo -e "    PCAP dir:  ${PCAP_DIR}"
echo -e "    Output:    ${OUTPUT_DIR}"
echo ""

# ---------------------------------------------------------------------------
# Extract SMTP sessions
# ---------------------------------------------------------------------------

EMAIL_COUNT=0
ATTACHMENT_COUNT=0

for pcap in "$PCAP_DIR"/*.pcap "$PCAP_DIR"/*.pcapng; do
    [[ -f "$pcap" ]] || continue
    basename_pcap=$(basename "$pcap")
    echo -e "${YELLOW}[*] Scanning: ${basename_pcap}${NC}"

    # Check for SMTP traffic
    has_smtp=$(tshark -r "$pcap" -Y "smtp || tcp.port == 25 || tcp.port == 587" -c 1 2>/dev/null | wc -l)
    if [[ "$has_smtp" -eq 0 ]]; then
        echo -e "    (no SMTP traffic)"
        continue
    fi

    # Export IMF (Internet Message Format) objects
    imf_dir="$OUTPUT_DIR/raw/${basename_pcap%.pcap*}"
    mkdir -p "$imf_dir"
    tshark -r "$pcap" --export-objects "imf,$imf_dir" 2>/dev/null || true

    imf_count=$(find "$imf_dir" -type f 2>/dev/null | wc -l)
    if [[ $imf_count -gt 0 ]]; then
        echo -e "    ${GREEN}Found $imf_count email messages${NC}"
        EMAIL_COUNT=$((EMAIL_COUNT + imf_count))
    fi

    # Extract SMTP streams for raw analysis
    streams=$(tshark -r "$pcap" -Y "tcp.dstport == 25 && tcp.len > 0" \
        -T fields -e tcp.stream 2>/dev/null | sort -un)

    for stream_id in $streams; do
        stream_file="$OUTPUT_DIR/raw/smtp_stream_${stream_id}.txt"

        # Follow the TCP stream
        tshark -r "$pcap" -q -z "follow,tcp,ascii,${stream_id}" \
            > "$stream_file" 2>/dev/null || true

        if [[ ! -s "$stream_file" ]]; then
            rm -f "$stream_file"
            continue
        fi

        # Extract email metadata
        from=$(grep -i "^MAIL FROM:" "$stream_file" 2>/dev/null | head -1 | sed 's/MAIL FROM://i' | tr -d '<> ' || echo "unknown")
        to=$(grep -i "^RCPT TO:" "$stream_file" 2>/dev/null | head -1 | sed 's/RCPT TO://i' | tr -d '<> ' || echo "unknown")
        subject=$(grep -i "^Subject:" "$stream_file" 2>/dev/null | head -1 | sed 's/Subject: //i' || echo "(no subject)")

        if [[ -n "$from" || -n "$to" ]]; then
            # Write summary
            {
                echo "Stream: $stream_id"
                echo "From: $from"
                echo "To: $to"
                echo "Subject: $subject"
                echo "File: $stream_file"
                echo "---"
            } >> "$OUTPUT_DIR/summary/email_index.txt"
        fi
    done

    # Also check port 587 (submission)
    streams_587=$(tshark -r "$pcap" -Y "tcp.dstport == 587 && tcp.len > 0" \
        -T fields -e tcp.stream 2>/dev/null | sort -un)

    for stream_id in $streams_587; do
        stream_file="$OUTPUT_DIR/raw/smtp_587_stream_${stream_id}.txt"
        tshark -r "$pcap" -q -z "follow,tcp,ascii,${stream_id}" \
            > "$stream_file" 2>/dev/null || true

        [[ -s "$stream_file" ]] || rm -f "$stream_file"
    done

    echo ""
done

# ---------------------------------------------------------------------------
# Parse extracted emails for attachments
# ---------------------------------------------------------------------------

echo -e "${YELLOW}[*] Scanning for attachments...${NC}"

for email_file in "$OUTPUT_DIR"/raw/*/*.eml "$OUTPUT_DIR"/raw/*/email_* 2>/dev/null; do
    [[ -f "$email_file" ]] || continue

    # Look for MIME boundaries indicating attachments
    if grep -qi "Content-Disposition: attachment\|Content-Transfer-Encoding: base64" "$email_file" 2>/dev/null; then
        ATTACHMENT_COUNT=$((ATTACHMENT_COUNT + 1))
        filename=$(grep -i "filename=" "$email_file" 2>/dev/null | head -1 | sed 's/.*filename="\?\([^"]*\)"\?.*/\1/' || echo "unknown")
        echo -e "    Attachment found: ${filename} in $(basename $email_file)"
    fi
done

# ---------------------------------------------------------------------------
# Extract SMTP auth credentials
# ---------------------------------------------------------------------------

echo -e "${YELLOW}[*] Scanning for SMTP credentials...${NC}"

cred_file="$OUTPUT_DIR/summary/smtp_credentials.txt"
for stream_file in "$OUTPUT_DIR"/raw/smtp_*.txt; do
    [[ -f "$stream_file" ]] || continue

    # Look for AUTH LOGIN or AUTH PLAIN
    if grep -q "AUTH LOGIN\|AUTH PLAIN" "$stream_file" 2>/dev/null; then
        echo -e "    ${GREEN}SMTP auth found in $(basename $stream_file)${NC}"
        grep -A 3 "AUTH" "$stream_file" >> "$cred_file" 2>/dev/null || true
        echo "---" >> "$cred_file"
    fi
done

# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------

echo ""
echo -e "${GREEN}[+] Email extraction complete${NC}"
echo -e "    Emails found:       ${EMAIL_COUNT}"
echo -e "    Attachments found:  ${ATTACHMENT_COUNT}"
echo ""

if [[ -f "$OUTPUT_DIR/summary/email_index.txt" ]]; then
    echo -e "${CYAN}[*] Email index:${NC}"
    head -30 "$OUTPUT_DIR/summary/email_index.txt" | sed 's/^/    /'
    total_indexed=$(grep -c "^Stream:" "$OUTPUT_DIR/summary/email_index.txt" 2>/dev/null || echo 0)
    if [[ $total_indexed -gt 5 ]]; then
        echo -e "    ... ($total_indexed total — see $OUTPUT_DIR/summary/email_index.txt)"
    fi
fi

if [[ -f "$cred_file" ]]; then
    echo ""
    echo -e "${GREEN}[+] SMTP credentials saved to: $cred_file${NC}"
    echo -e "    (base64 encoded — decode with: echo '<value>' | base64 -d)"
fi