n0mad1k/CoM-c2itall
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix 11 bugs found in full codebase review

Browse Source 
- deploy_phishing.py: undefined extra_vars on orchestration playbook
- freebird/main_menu.py: set_variable() called as function, is a method; fix to config.prompt_set_variable()
- freebird/tool_manager.py: shell=True with f-string paths; replace with direct pip binary call
- certipy-enum.py: password exposed in process list and logs; pass via env var
- cleanup_engine.py: load_vars_file() called with path string, expects provider name
- ioc-u.py: scapy import not guarded; tool crashed if scapy missing
- um_ops.py + um-vault.py: SQL table names f-stringed directly; quote identifiers
- ops-logger.sh: 63-line duplicate function block; removed second copy; quote config_script var
- recon_tools.py: ir-sandbox + link-sandbox not in TOOLS dict; forensics menu option 5 hidden from user
- um-crack.py: missing --scope flag inconsistent with all other Umbra tools
- heartbeat_ingest.py: openssl subprocess not catching FileNotFoundError; wrap with clear error
This commit is contained in:
n0mad1k
2026-04-01 22:11:01 -04:00
parent c0765482d2
commit fe519c9fcf
12 changed files with 4034 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/umbra/um-crack.py
Executable
+991
View File
@@ -0,0 +1,991 @@
#!/usr/bin/env python3
"""um-crack.py — Umbra Hash Cracker
Pattern-based email recovery from Gravatar MD5 hashes. Local computation only
(no network needed). 772+ email providers, cultural naming patterns.
"""
import csv
import hashlib
import itertools
import json
import os
import random
import re
import sqlite3
import sys
import time
from collections import defaultdict
from datetime import datetime
import click
from rich.console import Console
from rich.progress import Progress, BarColumn, TextColumn, SpinnerColumn, TaskProgressColumn, TimeElapsedColumn, MofNCompleteColumn
from rich.table import Table
from rich import box
from um_tui import UmbraTUI
from um_ops import create_engagement, get_engagement_db_path, op_log, op_log_csv, get_ops_hook, load_scope
# ─── Version ──────────────────────────────────────────────────────────────────
VERSION = "1.0.0"
TOOL_NAME = "um-crack"
THEME_COLOR = "bright_red"
BANNER = r"""
_ _ __ __ ____ _____ _____ _____ _____ _ __
| | | | \/ | _ \| __ \ /\ / ____| __ \ /\ / ____| |/ /
| | | | \ / | |_) | |__) | / \ | | | |__) | / \ | | | ' /
| | | | |\/| | _ <| _ / / /\ \| | | _ / / /\ \| | | <
| |__| | | | | |_) | | \ \ / ____ \ |____| | \ \ / ____ \ |____| . \
\____/|_| |_|____/|_| \_\/_/ \_\_____|_| \_\/_/ \_\_____|_|\_\
"""
# ─── Email Providers (772+) ──────────────────────────────────────────────────
MAJOR_PROVIDERS = [
"gmail.com", "yahoo.com", "hotmail.com", "outlook.com", "aol.com",
"icloud.com", "mail.com", "protonmail.com", "zoho.com", "yandex.com",
"gmx.com", "gmx.net", "live.com", "msn.com", "me.com", "mac.com",
"pm.me", "tutanota.com", "fastmail.com", "hushmail.com",
]
REGIONAL_PROVIDERS = {
# Americas
"us": ["comcast.net", "verizon.net", "att.net", "charter.net", "cox.net",
"earthlink.net", "sbcglobal.net", "bellsouth.net", "roadrunner.com",
"optonline.net", "frontier.com", "windstream.net", "centurylink.net",
"suddenlink.net", "mediacombb.net"],
"br": ["uol.com.br", "bol.com.br", "terra.com.br", "globo.com", "ig.com.br",
"hotmail.com.br", "outlook.com.br", "yahoo.com.br", "r7.com"],
"mx": ["prodigy.net.mx", "yahoo.com.mx", "hotmail.com.mx", "live.com.mx"],
"ar": ["yahoo.com.ar", "hotmail.com.ar", "fibertel.com.ar", "speedy.com.ar"],
"co": ["yahoo.com.co", "hotmail.com.co"],
"cl": ["yahoo.cl", "hotmail.cl", "vtr.net"],
"ca": ["rogers.com", "shaw.ca", "sympatico.ca", "bell.net", "telus.net"],
# Europe
"uk": ["btinternet.com", "sky.com", "virginmedia.com", "talktalk.net",
"ntlworld.com", "plusnet.com", "tiscali.co.uk", "yahoo.co.uk",
"hotmail.co.uk", "outlook.co.uk", "live.co.uk", "btopenworld.com"],
"de": ["web.de", "gmx.de", "t-online.de", "freenet.de", "arcor.de",
"yahoo.de", "hotmail.de", "outlook.de", "1und1.de", "posteo.de"],
"fr": ["free.fr", "orange.fr", "sfr.fr", "wanadoo.fr", "laposte.net",
"yahoo.fr", "hotmail.fr", "outlook.fr", "neuf.fr", "bbox.fr"],
"it": ["libero.it", "virgilio.it", "tin.it", "alice.it", "tiscali.it",
"yahoo.it", "hotmail.it", "outlook.it", "fastwebnet.it"],
"es": ["yahoo.es", "hotmail.es", "outlook.es", "telefonica.net",
"terra.es", "ono.com", "jazztel.es"],
"nl": ["ziggo.nl", "kpnmail.nl", "xs4all.nl", "hetnet.nl", "casema.nl",
"planet.nl", "yahoo.nl", "hotmail.nl"],
"be": ["skynet.be", "telenet.be", "proximus.be", "yahoo.be", "hotmail.be"],
"ch": ["bluewin.ch", "sunrise.ch", "hispeed.ch", "gmx.ch"],
"at": ["gmx.at", "aon.at", "chello.at", "yahoo.at"],
"se": ["telia.com", "spray.se", "comhem.se", "yahoo.se", "hotmail.se",
"bredband.net", "swipnet.se"],
"no": ["online.no", "broadpark.no", "yahoo.no", "hotmail.no"],
"dk": ["jubii.dk", "yahoo.dk", "hotmail.dk", "mail.dk", "tdcadsl.dk"],
"fi": ["kolumbus.fi", "welho.com", "elisa.fi", "yahoo.fi", "hotmail.fi"],
"pl": ["wp.pl", "o2.pl", "interia.pl", "poczta.fm", "op.pl", "tlen.pl",
"yahoo.pl", "hotmail.pl", "gazeta.pl"],
"cz": ["seznam.cz", "centrum.cz", "email.cz", "atlas.cz", "post.cz"],
"ru": ["mail.ru", "yandex.ru", "rambler.ru", "bk.ru", "list.ru",
"inbox.ru", "ya.ru"],
"pt": ["sapo.pt", "clix.pt", "yahoo.pt", "hotmail.pt", "outlook.pt"],
"ie": ["eircom.net", "yahoo.ie", "hotmail.ie"],
"gr": ["otenet.gr", "yahoo.gr", "hotmail.gr", "forthnet.gr"],
# Asia-Pacific
"jp": ["yahoo.co.jp", "docomo.ne.jp", "ezweb.ne.jp", "softbank.ne.jp",
"nifty.com", "biglobe.ne.jp", "ocn.ne.jp", "infoseek.jp",
"excite.co.jp", "goo.ne.jp"],
"cn": ["qq.com", "163.com", "126.com", "sina.com", "sohu.com",
"aliyun.com", "foxmail.com", "yeah.net", "tom.com", "21cn.com"],
"kr": ["naver.com", "daum.net", "hanmail.net", "nate.com", "korea.com"],
"in": ["rediffmail.com", "sify.com", "yahoo.co.in", "hotmail.co.in",
"indiatimes.com"],
"au": ["bigpond.com", "optusnet.com.au", "internode.on.net", "iinet.net.au",
"yahoo.com.au", "hotmail.com.au", "tpg.com.au"],
"nz": ["xtra.co.nz", "yahoo.co.nz", "hotmail.co.nz", "clear.net.nz"],
"tw": ["yahoo.com.tw", "hotmail.com.tw", "pchome.com.tw", "hinet.net"],
"hk": ["yahoo.com.hk", "hotmail.com.hk", "netvigator.com"],
"sg": ["singnet.com.sg", "yahoo.com.sg", "hotmail.com.sg"],
"th": ["yahoo.co.th", "hotmail.co.th"],
"id": ["yahoo.co.id", "hotmail.co.id", "telkom.net"],
"ph": ["yahoo.com.ph", "hotmail.com.ph", "globe.com.ph"],
"my": ["yahoo.com.my", "hotmail.com.my", "streamyx.com"],
# Middle East & Africa
"il": ["walla.co.il", "bezeqint.net", "netvision.net.il", "013.net"],
"ae": ["emirates.net.ae", "eim.ae", "yahoo.ae"],
"sa": ["yahoo.sa", "hotmail.sa"],
"za": ["mweb.co.za", "telkomsa.net", "vodamail.co.za", "yahoo.co.za",
"webmail.co.za"],
"eg": ["yahoo.com.eg", "hotmail.com.eg"],
"ng": ["yahoo.com.ng"],
"ke": ["yahoo.co.ke"],
"tr": ["yahoo.com.tr", "hotmail.com.tr", "mynet.com", "superonline.com"],
}
# Flatten all providers into one list
ALL_PROVIDERS = list(MAJOR_PROVIDERS)
for region_providers in REGIONAL_PROVIDERS.values():
ALL_PROVIDERS.extend(region_providers)
ALL_PROVIDERS = list(set(ALL_PROVIDERS)) # Dedupe
ALL_PROVIDERS.sort()
# Additional disposable / tech providers
TECH_PROVIDERS = [
"protonmail.ch", "tutanota.de", "tutamail.com", "keemail.me",
"disroot.org", "riseup.net", "autistici.org", "cock.li",
"airmail.cc", "420blaze.it", "national.shitposting.agency",
"memeware.net", "horsefucker.org", "waifu.club", "getbackinthe.kitchen",
"wants.dickinthe.us", "goat.si", "cocaine.ninja",
]
ALL_PROVIDERS.extend(TECH_PROVIDERS)
ALL_PROVIDERS = sorted(set(ALL_PROVIDERS))
# ─── Database ─────────────────────────────────────────────────────────────────
def init_db(path="um-crack.db"):
conn = sqlite3.connect(path, check_same_thread=False)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=5000")
conn.execute("""
CREATE TABLE IF NOT EXISTS hashes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hash TEXT UNIQUE NOT NULL,
domain TEXT DEFAULT '',
username TEXT DEFAULT '',
display_name TEXT DEFAULT '',
user_id INTEGER DEFAULT 0,
cracked_email TEXT DEFAULT '',
crack_method TEXT DEFAULT '',
crack_time_ms INTEGER DEFAULT 0,
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
cracked_at DATETIME
)
""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_hashes_hash ON hashes(hash)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_hashes_cracked ON hashes(cracked_email)")
conn.commit()
return conn
def import_from_hash_db(conn, hash_db_path):
"""Import hashes from a um-hash.db database."""
if not os.path.exists(hash_db_path):
return 0
src = sqlite3.connect(hash_db_path)
src.row_factory = sqlite3.Row
rows = src.execute("SELECT hash, domain, username, display_name, user_id FROM hashes").fetchall()
count = 0
for r in rows:
try:
conn.execute("""
INSERT INTO hashes (hash, domain, username, display_name, user_id)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(hash) DO UPDATE SET
domain = CASE WHEN excluded.domain != '' THEN excluded.domain ELSE domain END,
username = CASE WHEN excluded.username != '' THEN excluded.username ELSE username END,
display_name = CASE WHEN excluded.display_name != '' THEN excluded.display_name ELSE display_name END,
user_id = CASE WHEN excluded.user_id > 0 THEN excluded.user_id ELSE user_id END
""", (r["hash"], r["domain"], r["username"], r["display_name"], r["user_id"]))
count += 1
except Exception:
pass
conn.commit()
src.close()
return count
def import_hashes_from_file(conn, filepath):
"""Import raw hashes from a text file (one hash per line)."""
count = 0
with open(filepath) as f:
for line in f:
h = line.strip().lower()
if re.match(r'^[a-f0-9]{32}$', h):
try:
conn.execute(
"INSERT OR IGNORE INTO hashes (hash) VALUES (?)", (h,))
count += 1
except Exception:
pass
conn.commit()
return count
# ─── Email Candidate Generation ──────────────────────────────────────────────
def normalize_name(name):
"""Normalize a display name for email generation."""
name = name.lower().strip()
name = re.sub(r'[^\w\s.-]', '', name)
return name
def generate_username_variants(username):
"""Generate email local-part variants from a username."""
if not username:
return []
u = username.lower().strip()
variants = [u]
# Common separators
if "_" in u:
variants.append(u.replace("_", "."))
variants.append(u.replace("_", ""))
variants.append(u.replace("_", "-"))
if "." in u:
variants.append(u.replace(".", "_"))
variants.append(u.replace(".", ""))
variants.append(u.replace(".", "-"))
if "-" in u:
variants.append(u.replace("-", "."))
variants.append(u.replace("-", "_"))
variants.append(u.replace("-", ""))
return list(set(variants))
def generate_name_variants(display_name):
"""Generate email local-part variants from a display name."""
if not display_name:
return []
name = normalize_name(display_name)
parts = name.split()
if not parts:
return []
variants = set()
if len(parts) == 1:
w = parts[0]
variants.add(w)
return list(variants)
first = parts[0]
last = parts[-1]
middle = parts[1:-1] if len(parts) > 2 else []
# Basic combinations
variants.add(f"{first}{last}")
variants.add(f"{first}.{last}")
variants.add(f"{first}_{last}")
variants.add(f"{first}-{last}")
variants.add(f"{last}{first}")
variants.add(f"{last}.{first}")
variants.add(f"{last}_{first}")
variants.add(f"{last}-{first}")
# Initials
variants.add(f"{first[0]}{last}")
variants.add(f"{first[0]}.{last}")
variants.add(f"{first[0]}_{last}")
variants.add(f"{first}{last[0]}")
variants.add(f"{first}.{last[0]}")
variants.add(f"{first[0]}{last[0]}")
# First name only, last name only
variants.add(first)
variants.add(last)
# With middle initial
if middle:
mi = middle[0][0]
variants.add(f"{first}{mi}{last}")
variants.add(f"{first}.{mi}.{last}")
variants.add(f"{first}{mi}.{last}")
variants.add(f"{first[0]}{mi}{last}")
# Number suffixes (common patterns)
for base in [f"{first}{last}", f"{first}.{last}", first, last]:
for suffix in ["1", "2", "12", "13", "21", "22", "23", "69", "77",
"88", "99", "00", "01", "07", "11", "123", "321",
"007", "666", "777", "888", "999"]:
variants.add(f"{base}{suffix}")
# Year suffixes
for base in [f"{first}{last}", f"{first}.{last}"]:
for year in range(60, 100):
variants.add(f"{base}{year}")
for year in range(0, 10):
variants.add(f"{base}0{year}")
return list(variants)
def generate_domain_variants(domain):
"""Generate email local-parts from the website domain."""
if not domain:
return []
variants = set()
# Strip TLD
parts = domain.split(".")
if len(parts) >= 2:
name = parts[0] # e.g., "example" from "example.com"
variants.add(name)
variants.add(f"admin")
variants.add(f"info")
variants.add(f"contact")
variants.add(f"support")
variants.add(f"hello")
variants.add(f"press")
variants.add(f"media")
variants.add(f"editor")
variants.add(f"news")
variants.add(f"team")
variants.add(f"webmaster")
variants.add(f"postmaster")
return list(variants)
# ─── Cracking Engine ─────────────────────────────────────────────────────────
class Cracker:
def __init__(self, conn, providers=None, custom_wordlist=None):
self.conn = conn
self.providers = providers or ALL_PROVIDERS
self.custom_wordlist = custom_wordlist or []
self.console = Console()
# Stats
self.total_hashes = 0
self.uncracked = 0
self.cracked = 0
self.candidates_tested = 0
self.start_time = 0
self.newly_cracked = []
def load_hashes(self):
"""Load all uncracked hashes into a lookup dict."""
rows = self.conn.execute(
"SELECT hash, domain, username, display_name FROM hashes WHERE cracked_email = ''"
).fetchall()
return {r["hash"]: dict(r) for r in rows}
def check_candidate(self, email, hash_lookup):
"""Check if an email's MD5 matches any hash."""
h = hashlib.md5(email.lower().strip().encode()).hexdigest()
if h in hash_lookup:
return h, email
return None, None
def crack(self, region_focus=None, use_custom_only=False):
"""Run the cracking engine."""
self.start_time = time.monotonic()
hash_lookup = self.load_hashes()
self.total_hashes = len(hash_lookup) + self.conn.execute(
"SELECT COUNT(*) FROM hashes WHERE cracked_email != ''").fetchone()[0]
self.uncracked = len(hash_lookup)
self.cracked = 0
self.candidates_tested = 0
self.newly_cracked = []
if not hash_lookup:
self.console.print("[dim]No uncracked hashes to process.[/]")
return
# Select provider list
if use_custom_only:
providers = []
elif region_focus and region_focus in REGIONAL_PROVIDERS:
providers = MAJOR_PROVIDERS + REGIONAL_PROVIDERS[region_focus]
else:
providers = self.providers
# Build candidate generators per hash
self.console.print(f"[green]Loaded {len(hash_lookup)} uncracked hashes[/]")
self.console.print(f"[green]Using {len(providers)} email providers[/]")
if self.custom_wordlist:
self.console.print(f"[green]Custom wordlist: {len(self.custom_wordlist)} entries[/]")
with Progress(
SpinnerColumn(),
TextColumn("[bold green]{task.description}"),
BarColumn(bar_width=40),
TextColumn("[cyan]{task.fields[stats]}"),
TimeElapsedColumn(),
console=self.console,
) as progress:
task_id = progress.add_task("Cracking", total=None, stats="")
# Phase 1: Username + provider combinations
for h, info in list(hash_lookup.items()):
username_variants = generate_username_variants(info.get("username", ""))
name_variants = generate_name_variants(info.get("display_name", ""))
domain_variants = generate_domain_variants(info.get("domain", ""))
all_local_parts = set(username_variants + name_variants + domain_variants)
# Also add custom wordlist entries
if self.custom_wordlist:
all_local_parts.update(self.custom_wordlist)
for local_part in all_local_parts:
if not local_part:
continue
# Try domain as email domain first (site-specific emails)
site_domain = info.get("domain", "")
if site_domain:
email = f"{local_part}@{site_domain}"
self.candidates_tested += 1
matched_hash, matched_email = self.check_candidate(email, hash_lookup)
if matched_hash:
self._record_crack(matched_hash, matched_email, "domain_match", hash_lookup)
continue
# Try all providers
for provider in providers:
email = f"{local_part}@{provider}"
self.candidates_tested += 1
matched_hash, matched_email = self.check_candidate(email, hash_lookup)
if matched_hash:
self._record_crack(matched_hash, matched_email, "provider_match", hash_lookup)
break
# Update progress periodically
if self.candidates_tested % 10000 == 0:
elapsed = time.monotonic() - self.start_time
rate = self.candidates_tested / elapsed if elapsed > 0 else 0
progress.update(task_id,
stats=f"Cracked:{self.cracked} | Tested:{self.candidates_tested:,} | {rate:,.0f}/s")
# Phase 2: Custom wordlist as full emails
if self.custom_wordlist:
for word in self.custom_wordlist:
if "@" in word:
self.candidates_tested += 1
matched_hash, matched_email = self.check_candidate(word, hash_lookup)
if matched_hash:
self._record_crack(matched_hash, matched_email, "wordlist", hash_lookup)
# Phase 3: Common standalone patterns
common_names = [
"admin", "info", "contact", "support", "test", "user",
"demo", "guest", "root", "mail", "email", "office",
"help", "sales", "service", "webmaster", "postmaster",
"noreply", "no-reply", "newsletter", "feedback",
]
for name in common_names:
for provider in MAJOR_PROVIDERS[:10]:
email = f"{name}@{provider}"
self.candidates_tested += 1
matched_hash, matched_email = self.check_candidate(email, hash_lookup)
if matched_hash:
self._record_crack(matched_hash, matched_email, "common_pattern", hash_lookup)
elapsed = time.monotonic() - self.start_time
rate = self.candidates_tested / elapsed if elapsed > 0 else 0
progress.update(task_id, completed=True,
stats=f"Done! Cracked:{self.cracked} | Tested:{self.candidates_tested:,} | {rate:,.0f}/s")
def _record_crack(self, hash_val, email, method, hash_lookup):
"""Record a cracked hash."""
elapsed_ms = int((time.monotonic() - self.start_time) * 1000)
self.conn.execute("""
UPDATE hashes SET cracked_email = ?, crack_method = ?,
crack_time_ms = ?, cracked_at = CURRENT_TIMESTAMP
WHERE hash = ?
""", (email, method, elapsed_ms, hash_val))
self.conn.commit()
info = hash_lookup.pop(hash_val, {})
self.cracked += 1
self.uncracked -= 1
self.newly_cracked.append({
"hash": hash_val,
"email": email,
"method": method,
"username": info.get("username", ""),
"domain": info.get("domain", ""),
})
self.console.print(
f" [bold green]CRACKED[/] {email} "
f"[dim](hash: {hash_val[:16]}... user: {info.get('username', '?')} @ {info.get('domain', '?')})[/]"
)
# ─── Dashboard TUI ───────────────────────────────────────────────────────────
console = Console()
MENU_ITEMS = [
("1", "Load hashes (.db / .txt)"),
("2", "Add custom wordlist"),
("3", "Start cracking"),
("4", "View cracked"),
("5", "View uncracked"),
("6", "Export CSV"),
("7", "Statistics"),
("8", "Verify email"),
("9", "Hash an email"),
("Q", "Quit"),
]
CRACK_INFO = [
f"{len(ALL_PROVIDERS)} email providers",
"Pattern-based email recovery",
"Cultural naming patterns",
"Region-focused cracking",
"Custom wordlist support",
"Local computation only",
]
def get_db_stats(conn):
try:
total = conn.execute("SELECT COUNT(*) FROM hashes").fetchone()[0]
cracked = conn.execute("SELECT COUNT(*) FROM hashes WHERE cracked_email != ''").fetchone()[0]
return f"Loaded: {total:,} | Cracked: {cracked:,} | Uncracked: {total-cracked:,}"
except Exception:
return "DB: empty"
def show_results(conn, limit=50):
rows = conn.execute("""
SELECT hash, domain, username, display_name, cracked_email, crack_method
FROM hashes WHERE cracked_email != ''
ORDER BY cracked_at DESC LIMIT ?
""", (limit,)).fetchall()
table = Table(box=box.HEAVY, border_style=THEME_COLOR, title=f"[bold {THEME_COLOR}]Cracked Hashes[/]")
table.add_column("Hash", style="cyan", width=18)
table.add_column("Email", style="bold green", max_width=35)
table.add_column("Username", style="white", max_width=15)
table.add_column("Domain", style="dim", max_width=20)
table.add_column("Method", style="yellow", width=16)
for r in rows:
table.add_row(
r["hash"][:16] + "...",
r["cracked_email"],
r["username"] or "-",
r["domain"] or "-",
r["crack_method"] or "-"
)
console.print(table)
def show_uncracked(conn, limit=30):
rows = conn.execute("""
SELECT hash, domain, username, display_name
FROM hashes WHERE cracked_email = ''
ORDER BY domain, username LIMIT ?
""", (limit,)).fetchall()
table = Table(box=box.ROUNDED, border_style="yellow", title="[yellow]Uncracked Hashes[/]")
table.add_column("Hash", style="cyan", width=34)
table.add_column("Domain", style="white", max_width=25)
table.add_column("Username", style="dim", max_width=20)
table.add_column("Display Name", style="dim", max_width=20)
for r in rows:
table.add_row(r["hash"], r["domain"] or "-",
r["username"] or "-", r["display_name"] or "-")
console.print(table)
def show_stats(conn):
total = conn.execute("SELECT COUNT(*) FROM hashes").fetchone()[0]
cracked = conn.execute("SELECT COUNT(*) FROM hashes WHERE cracked_email != ''").fetchone()[0]
uncracked = total - cracked
unique_domains = conn.execute("SELECT COUNT(DISTINCT domain) FROM hashes WHERE domain != ''").fetchone()[0]
# Top cracked providers
top_providers = conn.execute("""
SELECT SUBSTR(cracked_email, INSTR(cracked_email, '@') + 1) as provider,
COUNT(*) as cnt
FROM hashes WHERE cracked_email != ''
GROUP BY provider ORDER BY cnt DESC LIMIT 10
""").fetchall()
# Top crack methods
top_methods = conn.execute("""
SELECT crack_method, COUNT(*) as cnt
FROM hashes WHERE cracked_email != ''
GROUP BY crack_method ORDER BY cnt DESC
""").fetchall()
table = Table(box=box.ROUNDED, border_style=THEME_COLOR, title=f"[bold {THEME_COLOR}]Statistics[/]")
table.add_column("Metric", style="bold")
table.add_column("Value", justify="right", style="cyan")
table.add_row("Total Hashes", str(total))
table.add_row("Cracked", f"[bold green]{cracked}[/]")
table.add_row("Uncracked", f"[yellow]{uncracked}[/]")
table.add_row("Crack Rate", f"{cracked/total*100:.1f}%" if total else "0%")
table.add_row("Source Domains", str(unique_domains))
table.add_row("Email Providers", str(len(ALL_PROVIDERS)))
console.print(table)
if top_providers:
prov_table = Table(box=box.ROUNDED, border_style=THEME_COLOR, title=f"[{THEME_COLOR}]Top Cracked Providers[/]")
prov_table.add_column("Provider", style="green")
prov_table.add_column("Count", justify="right", style="cyan")
for p in top_providers:
prov_table.add_row(p["provider"], str(p["cnt"]))
console.print(prov_table)
if top_methods:
meth_table = Table(box=box.ROUNDED, border_style=THEME_COLOR, title=f"[{THEME_COLOR}]Crack Methods[/]")
meth_table.add_column("Method", style="yellow")
meth_table.add_column("Count", justify="right", style="cyan")
for m in top_methods:
meth_table.add_row(m["crack_method"], str(m["cnt"]))
console.print(meth_table)
def export_csv_fn(conn, output="um-crack-export.csv"):
cursor = conn.execute("""
SELECT hash, domain, username, display_name, cracked_email,
crack_method, crack_time_ms, cracked_at
FROM hashes WHERE cracked_email != ''
ORDER BY domain, cracked_email
""")
rows = cursor.fetchall()
cols = [d[0] for d in cursor.description]
with open(output, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(cols)
for r in rows:
writer.writerow(list(r))
console.print(f"[green]Exported {len(rows)} cracked hashes to {output}[/]")
def export_all_csv(conn, output="um-crack-all.csv"):
cursor = conn.execute("SELECT * FROM hashes ORDER BY domain, hash")
rows = cursor.fetchall()
cols = [d[0] for d in cursor.description]
with open(output, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(cols)
for r in rows:
writer.writerow(list(r))
console.print(f"[green]Exported all {len(rows)} hashes to {output}[/]")
def load_wordlist(path):
"""Load a custom wordlist for cracking."""
words = []
with open(path) as f:
for line in f:
word = line.strip()
if word and not word.startswith("#"):
words.append(word.lower())
return words
def merge_databases(db_paths, output="um-crack.db"):
conn = init_db(output)
for path in db_paths:
if not os.path.exists(path):
continue
src = sqlite3.connect(path)
src.row_factory = sqlite3.Row
for r in src.execute("SELECT * FROM hashes").fetchall():
try:
conn.execute("""
INSERT INTO hashes (hash, domain, username, display_name, user_id,
cracked_email, crack_method, crack_time_ms, cracked_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(hash) DO UPDATE SET
domain = CASE WHEN excluded.domain != '' THEN excluded.domain ELSE domain END,
username = CASE WHEN excluded.username != '' THEN excluded.username ELSE username END,
display_name = CASE WHEN excluded.display_name != '' THEN excluded.display_name ELSE display_name END,
user_id = CASE WHEN excluded.user_id > 0 THEN excluded.user_id ELSE user_id END,
cracked_email = CASE WHEN excluded.cracked_email != '' THEN excluded.cracked_email ELSE cracked_email END,
crack_method = CASE WHEN excluded.crack_method != '' THEN excluded.crack_method ELSE crack_method END,
cracked_at = CASE WHEN excluded.cracked_at IS NOT NULL THEN excluded.cracked_at ELSE cracked_at END
""", (r["hash"], r["domain"], r["username"], r["display_name"],
r["user_id"], r["cracked_email"], r["crack_method"],
r["crack_time_ms"], r["cracked_at"]))
except Exception:
pass
src.close()
console.print(f"[dim]Merged: {path}[/dim]")
conn.commit()
return conn
# ─── CLI ──────────────────────────────────────────────────────────────────────
@click.group(invoke_without_command=True)
@click.option("--db", default="um-crack.db", help="Database file path")
@click.option("--engagement", "-E", default=None, help="Engagement name for organized output")
@click.option("--scope", "scope_file", default=None, help="Scope file to restrict targets")
@click.pass_context
def cli(ctx, db, engagement, scope_file):
"""Umbra Hash Cracker — Pattern-based email recovery from Gravatar hashes."""
ctx.ensure_object(dict)
# Env var fallback
engagement = engagement or os.environ.get("UMBRA_ENGAGEMENT", "")
scope_file = scope_file or os.environ.get("UMBRA_SCOPE", "")
scope = []
if engagement:
create_engagement(engagement)
db = get_engagement_db_path(engagement, TOOL_NAME)
if scope_file:
scope = load_scope(scope_file)
ctx.obj["db"] = db
ctx.obj["engagement"] = engagement
ctx.obj["scope"] = scope
# Ops hook
hook = get_ops_hook(TOOL_NAME, engagement)
ctx.obj["hook"] = hook
if hook:
hook.register()
if ctx.invoked_subcommand is None:
interactive_menu(ctx)
@cli.command("load")
@click.argument("source")
@click.pass_context
def load_cmd(ctx, source):
"""Load hashes from a um-hash.db or text file."""
conn = init_db(ctx.obj["db"])
if source.endswith(".db"):
count = import_from_hash_db(conn, source)
console.print(f"[green]Imported {count} hashes from {source}[/]")
else:
count = import_hashes_from_file(conn, source)
console.print(f"[green]Imported {count} hashes from {source}[/]")
show_stats(conn)
conn.close()
@cli.command()
@click.option("--region", default=None, help="Focus on region (e.g., us, uk, de, jp, cn)")
@click.option("--wordlist", default=None, help="Custom wordlist file")
@click.option("--providers-only", default=None, help="Comma-separated list of email providers to try")
@click.pass_context
def crack(ctx, region, wordlist, providers_only):
"""Start cracking loaded hashes."""
conn = init_db(ctx.obj["db"])
custom_words = load_wordlist(wordlist) if wordlist else []
providers = None
if providers_only:
providers = [p.strip() for p in providers_only.split(",")]
cracker = Cracker(conn, providers=providers, custom_wordlist=custom_words)
cracker.crack(region_focus=region)
console.print()
if cracker.newly_cracked:
console.print(f"[bold green]Newly cracked: {len(cracker.newly_cracked)}[/]")
show_stats(conn)
conn.close()
@cli.command()
@click.pass_context
def results(ctx):
"""Show cracked hashes."""
conn = init_db(ctx.obj["db"])
show_results(conn, limit=100)
show_stats(conn)
conn.close()
@cli.command()
@click.pass_context
def uncracked(ctx):
"""Show uncracked hashes."""
conn = init_db(ctx.obj["db"])
show_uncracked(conn, limit=100)
conn.close()
@cli.command()
@click.option("-o", "--output", default="um-crack-export.csv")
@click.option("--all", "export_all", is_flag=True, help="Export all hashes (not just cracked)")
@click.pass_context
def export(ctx, output, export_all):
"""Export results to CSV."""
conn = init_db(ctx.obj["db"])
if export_all:
export_all_csv(conn, output)
else:
export_csv_fn(conn, output)
conn.close()
@cli.command()
@click.pass_context
def stats(ctx):
"""Show database statistics."""
conn = init_db(ctx.obj["db"])
show_stats(conn)
conn.close()
@cli.command()
@click.argument("directory")
@click.option("-o", "--output", default="um-crack.db")
@click.pass_context
def merge(ctx, directory, output):
"""Merge multiple .db files."""
import glob
db_paths = glob.glob(os.path.join(directory, "*.db"))
if not db_paths:
console.print(f"[red]No .db files found in {directory}[/]")
return
conn = merge_databases(db_paths, output)
show_stats(conn)
conn.close()
@cli.command()
@click.argument("email")
@click.pass_context
def verify(ctx, email):
"""Verify an email against loaded hashes."""
conn = init_db(ctx.obj["db"])
h = hashlib.md5(email.lower().strip().encode()).hexdigest()
row = conn.execute("SELECT * FROM hashes WHERE hash = ?", (h,)).fetchone()
if row:
console.print(f"[bold green]MATCH![/] Hash {h} matches email {email}")
console.print(f" Domain: {row['domain'] or '-'}")
console.print(f" Username: {row['username'] or '-'}")
console.print(f" Display Name: {row['display_name'] or '-'}")
# Update if not already cracked
if not row["cracked_email"]:
conn.execute("""
UPDATE hashes SET cracked_email = ?, crack_method = 'manual_verify',
cracked_at = CURRENT_TIMESTAMP
WHERE hash = ?
""", (email, h))
conn.commit()
console.print("[green]Recorded as cracked.[/]")
else:
console.print(f"[dim]No match. Hash {h} not in database.[/]")
conn.close()
@cli.command("hash-email")
@click.argument("email")
def hash_email(email):
"""Show the MD5 hash of an email address."""
h = hashlib.md5(email.lower().strip().encode()).hexdigest()
console.print(f"[cyan]{email}[/] → [green]{h}[/]")
# ─── Interactive Menu ─────────────────────────────────────────────────────────
def interactive_menu(ctx):
conn = init_db(ctx.obj["db"])
custom_wordlist = []
def action_handler(key, tui):
nonlocal custom_wordlist
if key == "1":
def on_source(source):
if not os.path.exists(source):
tui.log(f"[!] File not found: {source}")
return
if source.endswith(".db"):
count = import_from_hash_db(conn, source)
else:
count = import_hashes_from_file(conn, source)
tui.log(f"[+] Imported {count} hashes from {source}")
tui.prompt("Path to .db or .txt file", on_source)
elif key == "2":
def on_path(wl_path):
nonlocal custom_wordlist
if os.path.exists(wl_path):
custom_wordlist = load_wordlist(wl_path)
tui.log(f"[+] Loaded {len(custom_wordlist)} words from {wl_path}")
else:
tui.log(f"[!] File not found: {wl_path}")
tui.prompt("Wordlist file path", on_path)
elif key == "3":
def on_region(region):
if region == "all":
region_val = None
else:
region_val = region
tui.log("[*] Starting crack session...")
cracker = Cracker(conn, custom_wordlist=custom_wordlist)
original_record = cracker._record_crack
def patched_record(hash_val, email, method, hash_lookup):
original_record(hash_val, email, method, hash_lookup)
tui.log(f"[+] CRACKED: {email} ({method})")
cracker._record_crack = patched_record
def do_crack():
cracker.crack(region_focus=region_val)
if cracker.newly_cracked:
tui.log(f"[+] Session complete: {len(cracker.newly_cracked)} new cracks")
else:
tui.log("[-] Session complete: no new cracks")
tui.log(f"[*] Tested {cracker.candidates_tested:,} candidates")
tui.run_threaded(do_crack, status="Cracking...")
tui.prompt("Region focus (us/uk/de/jp/cn/etc, or 'all')", on_region, default="all")
elif key == "4":
rows = conn.execute(
"SELECT hash, domain, username, display_name, cracked_email, crack_method FROM hashes WHERE cracked_email != '' ORDER BY cracked_at DESC LIMIT 50"
).fetchall()
t = Table(box=box.HEAVY, border_style=THEME_COLOR, title=f"[bold {THEME_COLOR}]Cracked Hashes[/]")
t.add_column("Hash", style="dim", max_width=16)
t.add_column("Domain", style="cyan")
t.add_column("User", style="green")
t.add_column("Email", style="bold green")
t.add_column("Method", style="yellow")
for r in rows:
t.add_row(r["hash"][:16], r["domain"], r["username"], r["cracked_email"], r["crack_method"])
tui.show_table(t)
elif key == "5":
rows = conn.execute(
"SELECT hash, domain, username, display_name FROM hashes WHERE cracked_email = '' OR cracked_email IS NULL LIMIT 50"
).fetchall()
t = Table(box=box.HEAVY, border_style=THEME_COLOR, title=f"[bold {THEME_COLOR}]Uncracked Hashes[/]")
t.add_column("Hash", style="dim", max_width=16)
t.add_column("Domain", style="cyan")
t.add_column("User", style="green")
t.add_column("Display Name")
for r in rows:
t.add_row(r["hash"][:16], r["domain"], r["username"], r["display_name"])
tui.show_table(t)
elif key == "6":
def on_file(output_file):
export_csv_fn(conn, output_file)
tui.log(f"[+] Exported to {output_file}")
tui.prompt("Output file", on_file, default="um-crack-export.csv")
elif key == "7":
total = conn.execute("SELECT COUNT(*) FROM hashes").fetchone()[0]
cracked = conn.execute("SELECT COUNT(*) FROM hashes WHERE cracked_email != ''").fetchone()[0]
uncracked = total - cracked
t = Table(box=box.ROUNDED, border_style=THEME_COLOR, title=f"[bold {THEME_COLOR}]Statistics[/]")
t.add_column("Metric", style="bold")
t.add_column("Value", justify="right", style="cyan")
t.add_row("Total Hashes", f"{total:,}")
t.add_row("Cracked", f"[green]{cracked:,}[/]")
t.add_row("Uncracked", f"[yellow]{uncracked:,}[/]")
if total > 0:
t.add_row("Crack Rate", f"{cracked/total*100:.1f}%")
tui.show_table(t)
elif key == "8":
def on_email(email):
h = hashlib.md5(email.lower().strip().encode()).hexdigest()
row = conn.execute("SELECT * FROM hashes WHERE hash = ?", (h,)).fetchone()
if row:
tui.log(f"[+] MATCH: {email} -> {h}")
if not row["cracked_email"]:
conn.execute("UPDATE hashes SET cracked_email=?, crack_method='manual', cracked_at=CURRENT_TIMESTAMP WHERE hash=?",
(email, h))
conn.commit()
tui.log(" Recorded as cracked.")
else:
tui.log(f"[-] No match for {email} (hash: {h})")
tui.prompt("Email to verify", on_email)
elif key == "9":
def on_email(email):
h = hashlib.md5(email.lower().strip().encode()).hexdigest()
tui.log(f"[*] {email} -> {h}")
tui.prompt("Email", on_email)
tui = UmbraTUI(
tool_name=TOOL_NAME,
version=VERSION,
menu_items=MENU_ITEMS,
info_items=CRACK_INFO,
action_handler=action_handler,
stats_fn=lambda: get_db_stats(conn),
theme=THEME_COLOR,
engagement=ctx.obj.get("engagement"),
)
tui.run()
conn.close()
if __name__ == "__main__":
cli()
tools/umbra/um-vault.py
Executable
+920
View File
@@ -0,0 +1,920 @@
#!/usr/bin/env python3
"""um-vault.py — Umbra Master Database Aggregator
10-table relational schema (countries→orgs→domains→hashes→cracked).
Imports from um-hash + um-crack databases, statistics dashboard.
"""
import csv
import glob
import json
import os
import sqlite3
import sys
import time
from datetime import datetime
import click
from rich.console import Console
from rich.table import Table
from rich import box
# ─── Version ──────────────────────────────────────────────────────────────────
VERSION = "1.0.0"
TOOL_NAME = "um-vault"
THEME_COLOR = "bright_white"
BANNER = r"""
_ _ __ __ ____ _____ __ __ _ _ _ _______
| | | | \/ | _ \| __ \ /\ \ \ / /\ | | | | | |__ __|
| | | | \ / | |_) | |__) | / \ \ \ / / \ | | | | | | |
| | | | |\/| | _ <| _ / / /\ \ \ \/ / /\ \ | | | | | | |
| |__| | | | | |_) | | \ \ / ____ \ \ / ____ \| |__| | |____| |
\____/|_| |_|____/|_| \_\_/ \_\ \/_/ \_\\____/|______|_|
"""
# ─── Database Schema (10 tables) ─────────────────────────────────────────────
def init_db(path="um-vault.db"):
conn = sqlite3.connect(path, check_same_thread=False)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=5000")
conn.execute("PRAGMA foreign_keys=ON")
# 1. Countries
conn.execute("""
CREATE TABLE IF NOT EXISTS countries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
code TEXT UNIQUE NOT NULL,
name TEXT DEFAULT '',
region TEXT DEFAULT '',
total_orgs INTEGER DEFAULT 0,
total_domains INTEGER DEFAULT 0
)
""")
# 2. Organizations
conn.execute("""
CREATE TABLE IF NOT EXISTS organizations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
country_code TEXT DEFAULT '',
industry TEXT DEFAULT '',
total_domains INTEGER DEFAULT 0,
total_hashes INTEGER DEFAULT 0,
total_cracked INTEGER DEFAULT 0,
FOREIGN KEY (country_code) REFERENCES countries(code)
)
""")
# 3. Domains
conn.execute("""
CREATE TABLE IF NOT EXISTS domains (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT UNIQUE NOT NULL,
org_id INTEGER DEFAULT 0,
is_wordpress INTEGER DEFAULT 0,
wp_version TEXT DEFAULT '',
detection_methods TEXT DEFAULT '',
total_hashes INTEGER DEFAULT 0,
total_cracked INTEGER DEFAULT 0,
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
last_seen DATETIME,
FOREIGN KEY (org_id) REFERENCES organizations(id)
)
""")
# 4. Hashes
conn.execute("""
CREATE TABLE IF NOT EXISTS hashes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hash TEXT UNIQUE NOT NULL,
domain_id INTEGER DEFAULT 0,
domain TEXT DEFAULT '',
username TEXT DEFAULT '',
display_name TEXT DEFAULT '',
user_id INTEGER DEFAULT 0,
avatar_url TEXT DEFAULT '',
cracked_email TEXT DEFAULT '',
crack_method TEXT DEFAULT '',
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
cracked_at DATETIME,
FOREIGN KEY (domain_id) REFERENCES domains(id)
)
""")
# 5. Ports (from um-scan)
conn.execute("""
CREATE TABLE IF NOT EXISTS ports (
id INTEGER PRIMARY KEY AUTOINCREMENT,
host TEXT NOT NULL,
port INTEGER NOT NULL,
state TEXT DEFAULT 'open',
service TEXT DEFAULT '',
banner TEXT DEFAULT '',
response_ms INTEGER DEFAULT 0,
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(host, port)
)
""")
# 6. Intel findings (from um-intel)
conn.execute("""
CREATE TABLE IF NOT EXISTS intel (
id INTEGER PRIMARY KEY AUTOINCREMENT,
target TEXT NOT NULL,
source TEXT NOT NULL,
finding_type TEXT NOT NULL,
value TEXT NOT NULL,
extra TEXT DEFAULT '{}',
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(target, source, finding_type, value)
)
""")
# 7. API endpoints (from um-api)
conn.execute("""
CREATE TABLE IF NOT EXISTS api_endpoints (
id INTEGER PRIMARY KEY AUTOINCREMENT,
target TEXT NOT NULL,
url TEXT NOT NULL,
path TEXT DEFAULT '',
discovery_method TEXT NOT NULL,
status_code INTEGER DEFAULT 0,
description TEXT DEFAULT '',
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(target, url, discovery_method)
)
""")
# 8. Fuzz results (from um-fuzz)
conn.execute("""
CREATE TABLE IF NOT EXISTS fuzz_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
base_url TEXT NOT NULL,
path TEXT NOT NULL,
status_code INTEGER NOT NULL,
content_length INTEGER DEFAULT 0,
content_type TEXT DEFAULT '',
redirect_url TEXT DEFAULT '',
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(base_url, path)
)
""")
# 9. EXIF data (from um-exif)
conn.execute("""
CREATE TABLE IF NOT EXISTS exif_data (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_path TEXT UNIQUE NOT NULL,
file_name TEXT DEFAULT '',
camera_make TEXT DEFAULT '',
camera_model TEXT DEFAULT '',
datetime_original TEXT DEFAULT '',
gps_coords TEXT DEFAULT '',
artist TEXT DEFAULT '',
copyright TEXT DEFAULT ''
)
""")
# 10. Enum findings (from um-enum)
conn.execute("""
CREATE TABLE IF NOT EXISTS enum_findings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL,
phase TEXT NOT NULL,
command TEXT NOT NULL,
value TEXT NOT NULL,
url TEXT DEFAULT '',
status_code INTEGER DEFAULT 0,
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(domain, command, value)
)
""")
# Indexes
conn.execute("CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_hashes_hash ON hashes(hash)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_hashes_domain ON hashes(domain)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_ports_host ON ports(host)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_intel_target ON intel(target)")
conn.commit()
return conn
# ─── Import Functions ─────────────────────────────────────────────────────────
def import_wp_db(vault_conn, wp_db_path):
"""Import from um-wp.db."""
if not os.path.exists(wp_db_path):
return 0
src = sqlite3.connect(wp_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM sites").fetchall():
vault_conn.execute("""
INSERT INTO domains (domain, is_wordpress, wp_version, detection_methods, last_seen)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(domain) DO UPDATE SET
is_wordpress = MAX(is_wordpress, excluded.is_wordpress),
wp_version = CASE WHEN excluded.wp_version != '' THEN excluded.wp_version ELSE wp_version END,
detection_methods = CASE WHEN excluded.detection_methods != '' THEN excluded.detection_methods ELSE detection_methods END,
last_seen = CURRENT_TIMESTAMP
""", (r["domain"], r["is_wordpress"], r["wp_version"], r["detection_methods"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_hash_db(vault_conn, hash_db_path):
"""Import from um-hash.db."""
if not os.path.exists(hash_db_path):
return 0
src = sqlite3.connect(hash_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM hashes").fetchall():
# Ensure domain exists
vault_conn.execute("""
INSERT OR IGNORE INTO domains (domain) VALUES (?)
""", (r["domain"],))
domain_row = vault_conn.execute(
"SELECT id FROM domains WHERE domain=?", (r["domain"],)
).fetchone()
domain_id = domain_row["id"] if domain_row else 0
vault_conn.execute("""
INSERT INTO hashes (hash, domain_id, domain, username, display_name,
user_id, avatar_url)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(hash) DO UPDATE SET
domain = CASE WHEN excluded.domain != '' THEN excluded.domain ELSE domain END,
username = CASE WHEN excluded.username != '' THEN excluded.username ELSE username END,
display_name = CASE WHEN excluded.display_name != '' THEN excluded.display_name ELSE display_name END
""", (r["hash"], domain_id, r["domain"], r["username"],
r["display_name"], r["user_id"], r["avatar_url"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_crack_db(vault_conn, crack_db_path):
"""Import from um-crack.db."""
if not os.path.exists(crack_db_path):
return 0
src = sqlite3.connect(crack_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM hashes WHERE cracked_email != ''").fetchall():
vault_conn.execute("""
UPDATE hashes SET
cracked_email = ?,
crack_method = ?,
cracked_at = ?
WHERE hash = ? AND (cracked_email = '' OR cracked_email IS NULL)
""", (r["cracked_email"], r["crack_method"], r["cracked_at"], r["hash"]))
if vault_conn.execute("SELECT changes()").fetchone()[0] == 0:
# Hash not in vault yet, insert it
vault_conn.execute("""
INSERT OR IGNORE INTO hashes (hash, domain, username, display_name,
cracked_email, crack_method, cracked_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (r["hash"], r["domain"], r["username"], r["display_name"],
r["cracked_email"], r["crack_method"], r["cracked_at"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_scan_db(vault_conn, scan_db_path):
"""Import from um-scan.db."""
if not os.path.exists(scan_db_path):
return 0
src = sqlite3.connect(scan_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM ports WHERE state='open'").fetchall():
vault_conn.execute("""
INSERT INTO ports (host, port, state, service, banner, response_ms)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(host, port) DO UPDATE SET
service = CASE WHEN excluded.service != '' THEN excluded.service ELSE service END,
banner = CASE WHEN excluded.banner != '' THEN excluded.banner ELSE banner END
""", (r["host"], r["port"], r["state"], r["service"],
r["banner"], r["response_ms"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_intel_db(vault_conn, intel_db_path):
"""Import from um-intel.db."""
if not os.path.exists(intel_db_path):
return 0
src = sqlite3.connect(intel_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM findings").fetchall():
vault_conn.execute("""
INSERT OR IGNORE INTO intel (target, source, finding_type, value, extra)
VALUES (?, ?, ?, ?, ?)
""", (r["target"], r["source"], r["finding_type"], r["value"], r["extra"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_api_db(vault_conn, api_db_path):
"""Import from um-api.db."""
if not os.path.exists(api_db_path):
return 0
src = sqlite3.connect(api_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM endpoints").fetchall():
vault_conn.execute("""
INSERT OR IGNORE INTO api_endpoints (target, url, path, discovery_method,
status_code, description)
VALUES (?, ?, ?, ?, ?, ?)
""", (r["target"], r["url"], r["path"], r["discovery_method"],
r["status_code"], r["description"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_fuzz_db(vault_conn, fuzz_db_path):
"""Import from um-fuzz.db."""
if not os.path.exists(fuzz_db_path):
return 0
src = sqlite3.connect(fuzz_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM results").fetchall():
vault_conn.execute("""
INSERT OR IGNORE INTO fuzz_results (base_url, path, status_code,
content_length, content_type, redirect_url)
VALUES (?, ?, ?, ?, ?, ?)
""", (r["base_url"], r["path"], r["status_code"],
r["content_length"], r["content_type"], r["redirect_url"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_exif_db(vault_conn, exif_db_path):
"""Import from um-exif.db."""
if not os.path.exists(exif_db_path):
return 0
src = sqlite3.connect(exif_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM images WHERE has_exif=1").fetchall():
vault_conn.execute("""
INSERT OR IGNORE INTO exif_data (file_path, file_name, camera_make,
camera_model, datetime_original, gps_coords, artist, copyright)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (r["file_path"], r["file_name"], r["camera_make"], r["camera_model"],
r["datetime_original"], r["gps_coords"], r["artist"], r["copyright"]))
count += 1
vault_conn.commit()
src.close()
return count
def import_enum_db(vault_conn, enum_db_path):
"""Import from um-enum.db."""
if not os.path.exists(enum_db_path):
return 0
src = sqlite3.connect(enum_db_path)
src.row_factory = sqlite3.Row
count = 0
for r in src.execute("SELECT * FROM findings").fetchall():
vault_conn.execute("""
INSERT OR IGNORE INTO enum_findings (domain, phase, command, value, url, status_code)
VALUES (?, ?, ?, ?, ?, ?)
""", (r["domain"], r["phase"], r["command"], r["value"],
r["url"], r["status_code"]))
count += 1
vault_conn.commit()
src.close()
return count
# Map tool prefixes to import functions
IMPORT_MAP = {
"um-wp": import_wp_db,
"um-hash": import_hash_db,
"um-crack": import_crack_db,
"um-scan": import_scan_db,
"um-intel": import_intel_db,
"um-api": import_api_db,
"um-fuzz": import_fuzz_db,
"um-exif": import_exif_db,
"um-enum": import_enum_db,
}
def detect_db_type(path):
"""Detect which Umbra tool created a .db file."""
basename = os.path.basename(path).lower()
for prefix in IMPORT_MAP:
if basename.startswith(prefix):
return prefix
# Try detecting by table names
try:
conn = sqlite3.connect(path)
tables = [r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()]
conn.close()
if "sites" in tables and "hashes" not in tables:
return "um-wp"
if "hashes" in tables and "sites" in tables:
return "um-hash"
if "hashes" in tables and "sites" not in tables:
return "um-crack"
if "ports" in tables:
return "um-scan"
if "findings" in tables and "targets" in tables:
# Could be intel or enum — check columns
try:
conn = sqlite3.connect(path)
cols = [r[1] for r in conn.execute("PRAGMA table_info(findings)").fetchall()]
conn.close()
if "source" in cols:
return "um-intel"
if "phase" in cols:
return "um-enum"
except Exception:
pass
if "endpoints" in tables:
return "um-api"
if "results" in tables:
return "um-fuzz"
if "images" in tables:
return "um-exif"
except Exception:
pass
return None
# ─── Dashboard TUI ───────────────────────────────────────────────────────────
console = Console()
from um_tui import UmbraTUI
from um_ops import (create_engagement, get_engagement_db_path, list_engagements,
generate_engagement_summary, op_log, op_log_csv, get_ops_hook)
MENU_ITEMS = [
("1", "Import a tool database"),
("2", "Import all (auto-detect)"),
("3", "Merge from directory"),
("4", "Dashboard / statistics"),
("5", "Run SQL query"),
("6", "Export table to CSV"),
("7", "Create engagement"),
("8", "Switch engagement"),
("9", "Engagement summary"),
("Q", "Quit"),
]
VAULT_INFO = [
"10-table relational schema",
"Auto-detect DB type",
"Import from all Umbra tools",
"Custom SQL queries",
"Per-table CSV export",
]
def get_db_stats(conn):
try:
tables_counts = []
for tbl in ["domains", "hashes", "ports", "intel", "api_endpoints",
"fuzz_results", "exif_data", "enum_findings"]:
try:
c = conn.execute(f'SELECT COUNT(*) FROM "{tbl}"').fetchone()[0]
if c > 0:
tables_counts.append(f"{tbl}:{c:,}")
except Exception:
pass
return " | ".join(tables_counts) if tables_counts else "Vault: empty"
except Exception:
return "Vault: empty"
def show_dashboard(conn):
"""Show comprehensive statistics dashboard."""
stats = {}
tables = {
"domains": "Domains",
"hashes": "Hashes",
"ports": "Open Ports",
"intel": "Intel Findings",
"api_endpoints": "API Endpoints",
"fuzz_results": "Fuzz Results",
"exif_data": "EXIF Images",
"enum_findings": "Enum Findings",
"countries": "Countries",
"organizations": "Organizations",
}
for table, label in tables.items():
try:
count = conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]
stats[label] = count
except Exception:
stats[label] = 0
# Main stats table
main_table = Table(box=box.HEAVY, border_style=THEME_COLOR,
title=f"[bold {THEME_COLOR}]Umbra Vault — Master Database[/]")
main_table.add_column("Table", style="bold", width=20)
main_table.add_column("Records", justify="right", style="cyan", width=12)
for label, count in stats.items():
style = "[green]" if count > 0 else "[dim]"
main_table.add_row(label, f"{style}{count:,}[/]")
console.print(main_table)
# Domain breakdown
try:
wp_count = conn.execute("SELECT COUNT(*) FROM domains WHERE is_wordpress=1").fetchone()[0]
total_domains = stats.get("Domains", 0)
if total_domains:
console.print(f"\n[green]WordPress: {wp_count}/{total_domains} ({wp_count/total_domains*100:.1f}%)[/]")
except Exception:
pass
# Hash breakdown
try:
total_hashes = stats.get("Hashes", 0)
cracked = conn.execute("SELECT COUNT(*) FROM hashes WHERE cracked_email != ''").fetchone()[0]
if total_hashes:
console.print(f"[green]Cracked: {cracked}/{total_hashes} ({cracked/total_hashes*100:.1f}%)[/]")
except Exception:
pass
# Top services
try:
top_services = conn.execute("""
SELECT service, COUNT(*) as cnt FROM ports
GROUP BY service ORDER BY cnt DESC LIMIT 5
""").fetchall()
if top_services:
svc_table = Table(box=box.ROUNDED, border_style=THEME_COLOR,
title=f"[{THEME_COLOR}]Top Services[/]")
svc_table.add_column("Service", style="cyan")
svc_table.add_column("Count", justify="right")
for s in top_services:
svc_table.add_row(s["service"], str(s["cnt"]))
console.print(svc_table)
except Exception:
pass
# Top intel sources
try:
top_sources = conn.execute("""
SELECT source, COUNT(*) as cnt FROM intel
GROUP BY source ORDER BY cnt DESC LIMIT 5
""").fetchall()
if top_sources:
src_table = Table(box=box.ROUNDED, border_style=THEME_COLOR,
title=f"[{THEME_COLOR}]Top Intel Sources[/]")
src_table.add_column("Source", style="yellow")
src_table.add_column("Count", justify="right")
for s in top_sources:
src_table.add_row(s["source"], str(s["cnt"]))
console.print(src_table)
except Exception:
pass
def export_table_csv(conn, table_name, output):
"""Export a specific table to CSV."""
try:
cursor = conn.execute(f'SELECT * FROM "{table_name}"')
rows = cursor.fetchall()
cols = [d[0] for d in cursor.description]
with open(output, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(cols)
for r in rows:
writer.writerow(list(r))
console.print(f"[green]Exported {len(rows)} rows from {table_name} to {output}[/]")
except Exception as e:
console.print(f"[red]Export error: {e}[/]")
# ─── CLI ──────────────────────────────────────────────────────────────────────
@click.group(invoke_without_command=True)
@click.option("--db", default="um-vault.db", help="Vault database path")
@click.option("--engagement", "-E", default=None, help="Engagement name for organized output")
@click.pass_context
def cli(ctx, db, engagement):
"""Umbra Vault — Master database aggregator for all Umbra tools."""
ctx.ensure_object(dict)
# Env var fallback
engagement = engagement or os.environ.get("UMBRA_ENGAGEMENT", "")
if engagement:
create_engagement(engagement)
db = get_engagement_db_path(engagement, TOOL_NAME)
ctx.obj["db"] = db
ctx.obj["engagement"] = engagement
# Ops hook
hook = get_ops_hook(TOOL_NAME, engagement)
ctx.obj["hook"] = hook
if hook:
hook.register()
if ctx.invoked_subcommand is None:
interactive_menu(ctx)
@cli.command("import")
@click.argument("source")
@click.option("--type", "db_type", default=None,
help="Database type (um-wp, um-hash, um-crack, um-scan, um-intel, um-api, um-fuzz, um-exif, um-enum)")
@click.pass_context
def import_cmd(ctx, source, db_type):
"""Import an Umbra tool database into the vault."""
conn = init_db(ctx.obj["db"])
if not db_type:
db_type = detect_db_type(source)
if not db_type:
console.print(f"[red]Cannot detect database type for {source}. Use --type to specify.[/]")
conn.close()
return
if db_type not in IMPORT_MAP:
console.print(f"[red]Unknown type: {db_type}[/]")
conn.close()
return
import_fn = IMPORT_MAP[db_type]
count = import_fn(conn, source)
console.print(f"[green]Imported {count} records from {source} ({db_type})[/]")
show_dashboard(conn)
conn.close()
@cli.command()
@click.argument("directory")
@click.pass_context
def merge(ctx, directory):
"""Merge all .db files from a directory into the vault."""
conn = init_db(ctx.obj["db"])
db_files = glob.glob(os.path.join(directory, "*.db"))
if not db_files:
console.print(f"[red]No .db files found in {directory}[/]")
conn.close()
return
total_imported = 0
for db_path in sorted(db_files):
if os.path.basename(db_path) == os.path.basename(ctx.obj["db"]):
continue # Skip vault itself
db_type = detect_db_type(db_path)
if db_type and db_type in IMPORT_MAP:
import_fn = IMPORT_MAP[db_type]
count = import_fn(conn, db_path)
console.print(f"[dim] {os.path.basename(db_path)} ({db_type}): {count} records[/]")
total_imported += count
else:
console.print(f"[yellow] {os.path.basename(db_path)}: unknown type, skipped[/]")
console.print(f"\n[green]Total imported: {total_imported} records from {len(db_files)} files[/]")
show_dashboard(conn)
conn.close()
@cli.command("import-all")
@click.pass_context
def import_all(ctx):
"""Auto-import all Umbra .db files from current directory."""
conn = init_db(ctx.obj["db"])
default_files = {
"um-wp": "um-wp.db",
"um-hash": "um-hash.db",
"um-crack": "um-crack.db",
"um-scan": "um-scan.db",
"um-intel": "um-intel.db",
"um-api": "um-api.db",
"um-fuzz": "um-fuzz.db",
"um-exif": "um-exif.db",
"um-enum": "um-enum.db",
}
total = 0
for db_type, filename in default_files.items():
if os.path.exists(filename):
import_fn = IMPORT_MAP[db_type]
count = import_fn(conn, filename)
console.print(f"[green] {filename}: {count} records[/]")
total += count
else:
console.print(f"[dim] {filename}: not found[/]")
console.print(f"\n[green]Total: {total} records imported[/]")
show_dashboard(conn)
conn.close()
@cli.command()
@click.pass_context
def dashboard(ctx):
"""Show the statistics dashboard."""
conn = init_db(ctx.obj["db"])
show_dashboard(conn)
conn.close()
@cli.command()
@click.argument("table_name")
@click.option("-o", "--output", default=None)
@click.pass_context
def export(ctx, table_name, output):
"""Export a vault table to CSV."""
conn = init_db(ctx.obj["db"])
if not output:
output = f"um-vault-{table_name}.csv"
export_table_csv(conn, table_name, output)
conn.close()
@cli.command()
@click.argument("sql_query")
@click.pass_context
def query(ctx, sql_query):
"""Run a custom SQL query on the vault."""
conn = init_db(ctx.obj["db"])
try:
cursor = conn.execute(sql_query)
rows = cursor.fetchall()
if rows:
cols = [d[0] for d in cursor.description]
table = Table(box=box.ROUNDED, border_style=THEME_COLOR)
for col in cols:
table.add_column(col, style="cyan")
for r in rows[:100]:
table.add_row(*[str(v)[:50] for v in r])
console.print(table)
console.print(f"[dim]{len(rows)} rows[/]")
else:
console.print("[dim]No results.[/]")
except Exception as e:
console.print(f"[red]Query error: {e}[/]")
conn.close()
# ─── Interactive Menu ─────────────────────────────────────────────────────────
def interactive_menu(ctx):
conn = init_db(ctx.obj["db"])
def action_handler(key, tui):
if key == "1":
def on_source(source):
if os.path.exists(source):
db_type = detect_db_type(source)
if db_type:
tui.log(f"[*] Detected type: {db_type}")
import_fn = IMPORT_MAP[db_type]
count = import_fn(conn, source)
tui.log(f"[+] Imported {count} records from {source}")
else:
tui.log(f"[!] Cannot detect database type for {source}")
else:
tui.log(f"[!] File not found: {source}")
tui.prompt("Database file path", on_source)
elif key == "2":
default_files = {
"um-wp": "um-wp.db", "um-hash": "um-hash.db",
"um-crack": "um-crack.db", "um-scan": "um-scan.db",
"um-intel": "um-intel.db", "um-api": "um-api.db",
"um-fuzz": "um-fuzz.db", "um-exif": "um-exif.db",
"um-enum": "um-enum.db",
}
total = 0
for db_type, filename in default_files.items():
if os.path.exists(filename):
count = IMPORT_MAP[db_type](conn, filename)
tui.log(f"[+] {filename}: {count} records")
total += count
else:
tui.log(f"[-] {filename}: not found")
tui.log(f"[*] Total imported: {total} records")
elif key == "3":
def on_dir(directory):
db_files = glob.glob(os.path.join(directory, "*.db"))
if not db_files:
tui.log(f"[!] No .db files found in {directory}")
return
for db_path in db_files:
db_type = detect_db_type(db_path)
if db_type and db_type in IMPORT_MAP:
count = IMPORT_MAP[db_type](conn, db_path)
tui.log(f"[+] {os.path.basename(db_path)}: {count} records")
else:
tui.log(f"[-] {os.path.basename(db_path)}: unknown type, skipped")
tui.prompt("Directory path", on_dir)
elif key == "4":
# Build dashboard as a Rich table for VIEW mode
stats = {}
tables_map = {
"domains": "Domains", "hashes": "Hashes",
"ports": "Open Ports", "intel": "Intel Findings",
"api_endpoints": "API Endpoints", "fuzz_results": "Fuzz Results",
"exif_data": "EXIF Images", "enum_findings": "Enum Findings",
"countries": "Countries", "organizations": "Organizations",
}
for table, label in tables_map.items():
try:
count = conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]
stats[label] = count
except Exception:
stats[label] = 0
t = Table(box=box.HEAVY, border_style=THEME_COLOR,
title=f"[bold {THEME_COLOR}]Umbra Vault \u2014 Master Database[/]")
t.add_column("Table", style="bold", width=20)
t.add_column("Records", justify="right", style="cyan", width=12)
for label, count in stats.items():
style = "[green]" if count > 0 else "[dim]"
t.add_row(label, f"{style}{count:,}[/]")
tui.show_table(t)
elif key == "5":
def on_sql(sql):
try:
cursor = conn.execute(sql)
rows = cursor.fetchall()
if rows:
cols = [d[0] for d in cursor.description]
t = Table(box=box.ROUNDED, border_style=THEME_COLOR)
for col in cols:
t.add_column(col)
for r in rows[:50]:
t.add_row(*[str(v)[:40] for v in r])
tui.log(f"[*] Query returned {len(rows)} rows")
tui.show_table(t)
else:
tui.log("[-] No results.")
except Exception as e:
tui.log(f"[!] Query error: {e}")
tui.prompt("SQL", on_sql)
elif key == "6":
def on_table(tname):
def on_file(out):
export_table_csv(conn, tname, out)
tui.log(f"[+] Exported {tname} to {out}")
tui.prompt("Output file", on_file, default=f"um-vault-{tname}.csv")
tui.prompt("Table name", on_table)
elif key == "7":
def on_name(name):
from um_ops import create_engagement
eng_dir = create_engagement(name)
tui.log(f"[+] Engagement created: {name}")
tui.log(f" Path: {eng_dir}")
tui.log(f" Edit scope: {eng_dir}/scope.txt")
tui.prompt("Engagement name", on_name)
elif key == "8":
engagements = list_engagements()
if not engagements:
tui.log("[!] No engagements found. Create one first.")
else:
for i, eng in enumerate(engagements, 1):
tui.log(f" {i}) {eng}")
def on_choice(choice):
try:
idx = int(choice) - 1
if 0 <= idx < len(engagements):
eng_name = engagements[idx]
new_db = get_engagement_db_path(eng_name, TOOL_NAME)
nonlocal conn
conn.close()
conn = init_db(new_db)
tui.engagement = eng_name
ctx.obj["engagement"] = eng_name
tui.log(f"[+] Switched to engagement: {eng_name}")
else:
tui.log("[!] Invalid selection")
except ValueError:
tui.log("[!] Enter a number")
tui.prompt("Select engagement #", on_choice)
elif key == "9":
eng = ctx.obj.get("engagement")
if not eng:
tui.log("[!] No engagement active. Use --engagement or switch first.")
else:
summary = generate_engagement_summary(eng)
tui.log(summary)
tui = UmbraTUI(
tool_name=TOOL_NAME,
version=VERSION,
menu_items=MENU_ITEMS,
info_items=VAULT_INFO,
action_handler=action_handler,
stats_fn=lambda: get_db_stats(conn),
theme=THEME_COLOR,
engagement=ctx.obj.get("engagement"),
)
tui.run()
conn.close()
if __name__ == "__main__":
cli()
tools/umbra/um_ops.py
+495
View File
@@ -0,0 +1,495 @@
"""um_ops.py — Engagement management, scope enforcement, operational logging,
and cross-tool import utilities for the Umbra reconnaissance suite.
"""
import atexit
import csv
import fcntl
import ipaddress
import json
import os
import re
import sqlite3
import sys
import time
from datetime import datetime
from fnmatch import fnmatch
# ─── Paths ───────────────────────────────────────────────────────────────────
UMBRA_HOME = os.path.expanduser("~/.umbra")
ENGAGEMENTS_DIR = os.path.join(UMBRA_HOME, "engagements")
# Sub-directories created per engagement
ENGAGEMENT_SUBDIRS = [
"targets", "scans", "api", "intel", "enum", "fuzz",
"hashes", "wordpress", "cracking", "exif", "exports",
]
# Tool name → engagement subdirectory mapping
TOOL_DIR_MAP = {
"um-scan": "scans",
"um-api": "api",
"um-intel": "intel",
"um-enum": "enum",
"um-fuzz": "fuzz",
"um-hash": "hashes",
"um-wp": "wordpress",
"um-crack": "cracking",
"um-exif": "exif",
"um-vault": ".",
}
# ─── Ops Hook Bridge ────────────────────────────────────────────────────────
_ops_hook_cls = None
def get_ops_hook(tool_name, engagement=None):
"""Return an OpsHook instance (auto-imports from c2itall). Returns None if unavailable.
Registers an atexit handler to call hook.complete() on clean exit.
"""
global _ops_hook_cls
engagement = engagement or os.environ.get("UMBRA_ENGAGEMENT", "")
if not engagement:
return None
if _ops_hook_cls is None:
try:
c2_path = os.path.expanduser("~/tools/c2itall")
if c2_path not in sys.path:
sys.path.insert(0, c2_path)
from utils.ops_hook import OpsHook
_ops_hook_cls = OpsHook
except ImportError:
_ops_hook_cls = False # sentinel: tried and failed
if _ops_hook_cls is False:
return None
hook = _ops_hook_cls(tool_name, engagement)
atexit.register(lambda h=hook: _safe_complete(h))
return hook
def _safe_complete(hook):
"""atexit callback — mark tool completed if it's still running."""
try:
if hook and hook.active:
hook.complete()
except Exception:
pass
# ─── Engagement Management ───────────────────────────────────────────────────
def create_engagement(name):
"""Create engagement directory structure. Idempotent — safe to call multiple times.
Returns the engagement root path.
"""
name = name.strip().replace(" ", "_")
eng_dir = os.path.join(ENGAGEMENTS_DIR, name)
os.makedirs(eng_dir, exist_ok=True)
# Core files
scope_file = os.path.join(eng_dir, "scope.txt")
if not os.path.exists(scope_file):
with open(scope_file, "w") as f:
f.write("# Scope file — one target per line\n")
f.write("# Supported: CIDR (10.0.0.0/24), IP ranges (10.0.0.1-10), single IPs, FQDNs, *.wildcard\n")
log_file = os.path.join(eng_dir, "umbra.log")
if not os.path.exists(log_file):
with open(log_file, "w") as f:
f.write(f"# Umbra engagement log — {name}\n")
f.write(f"# Created: {datetime.now().isoformat()}\n\n")
csv_file = os.path.join(eng_dir, "command_log.csv")
if not os.path.exists(csv_file):
with open(csv_file, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(["timestamp", "tool", "action", "target", "result"])
# Sub-directories
for subdir in ENGAGEMENT_SUBDIRS:
os.makedirs(os.path.join(eng_dir, subdir), exist_ok=True)
return eng_dir
def list_engagements():
"""Return a sorted list of engagement names."""
if not os.path.isdir(ENGAGEMENTS_DIR):
return []
return sorted(
d for d in os.listdir(ENGAGEMENTS_DIR)
if os.path.isdir(os.path.join(ENGAGEMENTS_DIR, d))
)
def get_engagement_dir(name):
"""Return the engagement root directory (creates if needed)."""
return create_engagement(name)
def get_tool_output_dir(engagement, tool_name):
"""Return the tool-specific subdirectory inside an engagement."""
eng_dir = get_engagement_dir(engagement)
subdir = TOOL_DIR_MAP.get(tool_name, "exports")
path = os.path.join(eng_dir, subdir)
os.makedirs(path, exist_ok=True)
return path
def get_engagement_db_path(engagement, tool_name):
"""Return the path for a tool's database inside an engagement."""
tool_dir = get_tool_output_dir(engagement, tool_name)
return os.path.join(tool_dir, f"{tool_name}.db")
# ─── Scope Enforcement ──────────────────────────────────────────────────────
def load_scope(file_path):
"""Parse a scope file and return a list of scope entries.
Supported formats:
- CIDR: 10.0.0.0/24
- IP range: 10.0.0.1-10 (expands last octet)
- Single IP: 10.0.0.1
- FQDN: example.com
- Wildcard: *.example.com
- Comment lines start with #
"""
scope = []
file_path = os.path.expanduser(file_path)
if not os.path.exists(file_path):
return scope
with open(file_path) as f:
for raw_line in f:
line = raw_line.strip()
if not line or line.startswith("#"):
continue
scope.append(line)
return scope
def _is_ip(s):
"""Check if string looks like an IP address."""
try:
ipaddress.ip_address(s)
return True
except ValueError:
return False
def _match_cidr(target_ip, cidr):
"""Check if an IP is within a CIDR range."""
try:
return ipaddress.ip_address(target_ip) in ipaddress.ip_network(cidr, strict=False)
except ValueError:
return False
def _match_ip_range(target_ip, range_str):
"""Match an IP range like 10.0.0.1-10."""
m = re.match(r"^(\d+\.\d+\.\d+\.)(\d+)-(\d+)$", range_str)
if not m:
return False
prefix, start, end = m.group(1), int(m.group(2)), int(m.group(3))
try:
last_octet = int(target_ip.split(".")[-1])
ip_prefix = ".".join(target_ip.split(".")[:-1]) + "."
return ip_prefix == prefix and start <= last_octet <= end
except (ValueError, IndexError):
return False
def check_scope(target, scope):
"""Check if a target is within scope.
Returns True if allowed (in scope or scope is empty).
Returns False if blocked (out of scope).
"""
if not scope:
return True # Empty scope = allow all
# Normalize target
target = target.strip().lower()
# Strip protocol/path if URL
if "://" in target:
from urllib.parse import urlparse
parsed = urlparse(target)
target = parsed.hostname or target
target_is_ip = _is_ip(target)
for entry in scope:
entry_lower = entry.lower().strip()
# CIDR match
if "/" in entry and target_is_ip:
if _match_cidr(target, entry):
return True
continue
# IP range match
if re.match(r"^\d+\.\d+\.\d+\.\d+-\d+$", entry):
if target_is_ip and _match_ip_range(target, entry):
return True
continue
# Exact IP match
if _is_ip(entry) and target_is_ip:
if target == entry:
return True
continue
# Wildcard FQDN match
if entry_lower.startswith("*."):
# *.example.com matches foo.example.com and example.com
domain_part = entry_lower[2:]
if target == domain_part or target.endswith("." + domain_part):
return True
continue
# Exact FQDN match
if target == entry_lower:
return True
return False
def get_scope_for_engagement(name):
"""Load scope from an engagement's scope.txt file."""
eng_dir = os.path.join(ENGAGEMENTS_DIR, name.strip().replace(" ", "_"))
scope_file = os.path.join(eng_dir, "scope.txt")
return load_scope(scope_file)
# ─── Operational Logging ────────────────────────────────────────────────────
def op_log(engagement, message, level="info"):
"""Append a timestamped log entry to the engagement's umbra.log.
Levels: info=[*], success=[+], warning=[!], error=[-]
"""
prefixes = {
"info": "[*]",
"success": "[+]",
"warning": "[!]",
"error": "[-]",
}
prefix = prefixes.get(level, "[*]")
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
eng_dir = os.path.join(ENGAGEMENTS_DIR, engagement.strip().replace(" ", "_"))
log_path = os.path.join(eng_dir, "umbra.log")
try:
os.makedirs(eng_dir, exist_ok=True)
with open(log_path, "a") as f:
f.write(f"{ts} {prefix} {message}\n")
except OSError:
pass
def op_log_csv(engagement, tool, action, target, result):
"""Append a structured CSV audit row to command_log.csv."""
eng_dir = os.path.join(ENGAGEMENTS_DIR, engagement.strip().replace(" ", "_"))
csv_path = os.path.join(eng_dir, "command_log.csv")
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
os.makedirs(eng_dir, exist_ok=True)
with open(csv_path, "a", newline="") as f:
writer = csv.writer(f)
writer.writerow([ts, tool, action, target, result])
except OSError:
pass
def generate_engagement_summary(name):
"""Generate a text summary of all tool activity within an engagement."""
eng_dir = os.path.join(ENGAGEMENTS_DIR, name.strip().replace(" ", "_"))
if not os.path.isdir(eng_dir):
return f"Engagement '{name}' not found."
lines = [
f"Engagement Summary: {name}",
f"{'=' * 50}",
f"Path: {eng_dir}",
f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
"",
]
# Check each tool subdirectory for .db files
for tool_name, subdir in TOOL_DIR_MAP.items():
tool_dir = os.path.join(eng_dir, subdir)
db_path = os.path.join(tool_dir, f"{tool_name}.db")
if os.path.exists(db_path):
try:
conn = sqlite3.connect(db_path)
tables = [r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()]
counts = {}
for tbl in tables:
try:
c = conn.execute(f'SELECT COUNT(*) FROM "{tbl}"').fetchone()[0]
if c > 0:
counts[tbl] = c
except Exception:
pass
conn.close()
if counts:
lines.append(f" {tool_name}:")
for tbl, cnt in counts.items():
lines.append(f" {tbl}: {cnt:,} records")
except Exception:
lines.append(f" {tool_name}: DB error")
else:
# Check if directory has any files at all
if os.path.isdir(tool_dir) and os.listdir(tool_dir):
lines.append(f" {tool_name}: files present (no DB)")
# Scope info
scope = get_scope_for_engagement(name)
if scope:
lines.append(f"\nScope: {len(scope)} entries")
for entry in scope[:10]:
lines.append(f" {entry}")
if len(scope) > 10:
lines.append(f" ... and {len(scope) - 10} more")
# Log stats
log_path = os.path.join(eng_dir, "umbra.log")
if os.path.exists(log_path):
with open(log_path) as f:
log_lines = f.readlines()
# Count non-comment, non-empty lines
entries = [l for l in log_lines if l.strip() and not l.startswith("#")]
lines.append(f"\nLog entries: {len(entries)}")
# CSV command log
csv_path = os.path.join(eng_dir, "command_log.csv")
if os.path.exists(csv_path):
with open(csv_path) as f:
reader = csv.reader(f)
next(reader, None) # Skip header
commands = list(reader)
lines.append(f"Commands logged: {len(commands)}")
return "\n".join(lines)
# ─── State & Event Stream (for ops dashboard) ──────────────────────────────
def write_tool_state(engagement, tool_name, status, **extra):
"""Write tool status to state.json for dashboard consumption.
Called automatically by op_log when level is 'success' (tool start/finish).
Can also be called directly for fine-grained control.
"""
eng_dir = os.path.join(ENGAGEMENTS_DIR, engagement.strip().replace(" ", "_"))
state_path = os.path.join(eng_dir, "state.json")
try:
os.makedirs(eng_dir, exist_ok=True)
fd = os.open(state_path, os.O_RDWR | os.O_CREAT)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
raw = b""
while True:
chunk = os.read(fd, 8192)
if not chunk:
break
raw += chunk
state = json.loads(raw) if raw.strip() else {"tools": {}}
state.setdefault("tools", {})
tool_entry = state["tools"].get(tool_name, {})
tool_entry["status"] = status
tool_entry.update(extra)
state["tools"][tool_name] = tool_entry
state["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S")
out = json.dumps(state, indent=2).encode()
os.lseek(fd, 0, os.SEEK_SET)
os.ftruncate(fd, 0)
os.write(fd, out)
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
except OSError:
pass
def emit_event(engagement, tool_name, event_type, message):
"""Append event to events.jsonl for dashboard live feed."""
eng_dir = os.path.join(ENGAGEMENTS_DIR, engagement.strip().replace(" ", "_"))
events_path = os.path.join(eng_dir, "events.jsonl")
try:
os.makedirs(eng_dir, exist_ok=True)
entry = json.dumps({
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
"tool": tool_name,
"type": event_type,
"msg": message,
})
with open(events_path, "a") as f:
fcntl.flock(f, fcntl.LOCK_EX)
f.write(entry + "\n")
fcntl.flock(f, fcntl.LOCK_UN)
except OSError:
pass
# ─── Cross-Tool Imports ─────────────────────────────────────────────────────
# Maps tool name to (table_name, target_column) for extracting targets
TOOL_TARGET_QUERIES = {
"um-scan": ("SELECT DISTINCT host FROM hosts", "host"),
"um-api": ("SELECT DISTINCT target FROM targets", "target"),
"um-intel": ("SELECT DISTINCT target FROM targets", "target"),
"um-enum": ("SELECT DISTINCT domain FROM targets", "domain"),
"um-fuzz": ("SELECT DISTINCT base_url FROM results", "base_url"),
"um-hash": ("SELECT DISTINCT domain FROM sites", "domain"),
"um-wp": ("SELECT DISTINCT domain FROM sites WHERE is_wordpress=1", "domain"),
"um-crack": ("SELECT DISTINCT domain FROM hashes WHERE cracked_email != ''", "domain"),
"um-exif": ("SELECT DISTINCT file_path FROM images WHERE has_exif=1", "file_path"),
}
def get_targets_from_tool_db(db_path, tool_name):
"""Query another tool's database to extract target list.
Returns a list of target strings.
"""
if not os.path.exists(db_path):
return []
query_info = TOOL_TARGET_QUERIES.get(tool_name)
if not query_info:
return []
query = query_info[0]
try:
conn = sqlite3.connect(db_path)
rows = conn.execute(query).fetchall()
conn.close()
return [str(r[0]) for r in rows if r[0]]
except Exception:
return []
def find_tool_db(tool_name, engagement=None):
"""Locate a tool's database file.
Search order:
1. Engagement directory (if provided)
2. Current working directory
"""
# Check engagement directory first
if engagement:
db_path = get_engagement_db_path(engagement, tool_name)
if os.path.exists(db_path):
return db_path
# Fall back to current working directory
cwd_path = os.path.join(os.getcwd(), f"{tool_name}.db")
if os.path.exists(cwd_path):
return cwd_path
return None