diff --git a/redflare/modules/__init__.py b/redflare/modules/__init__.py new file mode 100644 index 0000000..d13655c --- /dev/null +++ b/redflare/modules/__init__.py @@ -0,0 +1,21 @@ +from .headers import HeaderModule +from .exposure import SensitiveExposureModule +from .mapping import ApplicationMappingModule +from .passive import PassiveReconModule +from .paths import PathDiscoveryModule +from .surface import SurfaceAnalysisModule +from .vulnerabilities import CVEIntelligenceModule +from .browser import NativeBrowserRuntimeModule +from .noauth import NativeNoAuthModule + +__all__ = [ + "ApplicationMappingModule", + "HeaderModule", + "PassiveReconModule", + "PathDiscoveryModule", + "SurfaceAnalysisModule", + "SensitiveExposureModule", + "CVEIntelligenceModule", + "NativeBrowserRuntimeModule", + "NativeNoAuthModule", +] diff --git a/redflare/modules/browser.py b/redflare/modules/browser.py new file mode 100644 index 0000000..7bda2a0 --- /dev/null +++ b/redflare/modules/browser.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio +import json +import time +from urllib.parse import urlsplit + +from redflare.core.models import Finding, ModuleResult, Target +from .base import Module, ModuleContext +from .headers import SECURITY_HEADERS +from .http import request + +INTERESTING = ("api", "graphql", "admin", "auth", "token", "upload", "config", "internal") + + +class NativeBrowserRuntimeModule(Module): + name = "browser_runtime" + description = "Natively capture browser/runtime requests, responses, redirects, console events, and application endpoints" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic(); result = ModuleResult(self.name, target.url) + try: + capture = asyncio.run(self._capture(target, context)) + except Exception as exc: + capture = self._fallback(target, context, f"{type(exc).__name__}: {exc}") + requests_seen = capture.get("requests", []); responses = capture.get("responses", []) + for item in requests_seen: + url = str(item.get("url") or ""); parsed = urlsplit(url) + if parsed.hostname == target.host: + context.surface_graph.add_endpoint(target.url, url, method=str(item.get("method") or "GET"), source=self.name, + authentication="observed" if item.get("authentication") else None) + headers = {k.lower(): v for k, v in (capture.get("main_headers") or {}).items()} + missing = [name for name in SECURITY_HEADERS if name not in headers] + if missing: + result.findings.append(Finding(context.run_id, target.url, self.name, "browser-security-headers", + "Browser-runtime response headers need hardening", "low", .95, + "Native runtime capture confirmed missing common security headers.", {"missing": missing}, ["browser", "headers", "native"])) + interesting = [item for item in requests_seen if any(term in str(item.get("url", "")).lower() for term in INTERESTING)] + if interesting: + result.findings.append(Finding(context.run_id, target.url, self.name, "browser-endpoints", + "Interesting endpoints observed during browser execution", "info", .8, + "Native runtime capture observed assessment-relevant endpoints.", {"endpoints": interesting[:50]}, ["browser", "endpoints", "native"])) + result.observations = {"engine": capture.get("engine"), "requests": len(requests_seen), "responses": len(responses), + "console_events": len(capture.get("console", [])), "final_url": capture.get("final_url"), + "fallback_reason": capture.get("fallback_reason")} + directory = context.artifact_dir / self.name / target.host; directory.mkdir(parents=True, exist_ok=True) + artifact = directory / "network_capture.json"; artifact.write_text(json.dumps(capture, indent=2), encoding="utf-8") + result.artifacts.append(str(artifact)); result.duration_seconds = round(time.monotonic() - started, 4); return result + + async def _capture(self, target: Target, context: ModuleContext) -> dict: + from playwright.async_api import async_playwright + captured = {"engine": "native-playwright", "target": target.url, "requests": [], "responses": [], "console": [], "main_headers": {}} + async with async_playwright() as runtime: + browser = await runtime.chromium.launch(headless=True) + browser_context = await browser.new_context(ignore_https_errors=True) + page = await browser_context.new_page() + page.on("request", lambda req: captured["requests"].append({"url": req.url, "method": req.method, "resource_type": req.resource_type})) + page.on("response", lambda res: captured["responses"].append({"url": res.url, "status": res.status})) + page.on("console", lambda msg: captured["console"].append({"type": msg.type, "text": msg.text[:500]})) + response = await page.goto(target.url, wait_until="domcontentloaded", timeout=int(max(5, context.timeout) * 1000)) + if response: captured["main_headers"] = await response.all_headers() + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(1500) + captured["final_url"] = page.url; captured["title"] = await page.title() + await page.screenshot(path=str(context.artifact_dir / f"{target.host}_runtime.png"), full_page=True) + await browser.close() + return captured + + @staticmethod + def _fallback(target: Target, context: ModuleContext, reason: str) -> dict: + response = request(target.url, context.timeout, max_body=500_000) + urls = context.surface_graph.request_urls(target.url) + return {"engine": "native-http-fallback", "fallback_reason": reason, "target": target.url, + "final_url": response.url, "main_headers": response.headers, + "requests": [{"url": url, "method": "GET", "resource_type": "mapped"} for url in urls] or [{"url": response.url, "method": "GET", "resource_type": "document"}], + "responses": [{"url": response.url, "status": response.status}], "console": []} diff --git a/redflare/modules/noauth.py b/redflare/modules/noauth.py new file mode 100644 index 0000000..197dc06 --- /dev/null +++ b/redflare/modules/noauth.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +import re +import time +from urllib.parse import urljoin + +from redflare.core.models import Finding, ModuleResult, Target +from .base import Module, ModuleContext +from .http import request + +PATHS = ("/", "/login", "/admin", "/dashboard", "/panel", "/console", "/manage", "/manager", + "/status", "/api/status", "/health", "/metrics", "/config", "/config.json", "/api/config", + "/graphql", "/swagger-ui.html", "/openapi.json", "/actuator/env", "/debug/vars", "/.env") +SENSITIVE = re.compile(r"(?:config|secret|credential|token|\.env|actuator|debug|metrics|console|admin|dashboard)", re.I) +AUTH = re.compile(r"(?:type=[\"']?password|/(?:login|signin|auth)(?:/|\?|$)|sign\s*in|log\s*in)", re.I) + + +class NativeNoAuthModule(Module): + name = "authorization_surface" + description = "Natively triage bounded in-scope web surfaces for missing authentication barriers" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic(); result = ModuleResult(self.name, target.url); hits = [] + for path in PATHS: + url = urljoin(target.url.rstrip("/") + "/", path.lstrip("/")) + try: + response = request(url, context.timeout, max_body=200_000, allowed_origin=(target.host, target.port)) + except Exception: + continue + body = response.body.decode("utf-8", errors="replace") + authenticated = response.status in {401, 403} or "www-authenticate" in response.headers or bool(AUTH.search(body)) + context.surface_graph.add_endpoint(target.url, response.url, method="GET", source=self.name, + content_type=response.headers.get("content-type"), status=response.status, + authentication="required" if authenticated else "not-observed") + if response.status == 200 and not authenticated and (path != "/" or SENSITIVE.search(body)) and SENSITIVE.search(path + " " + body[:5000]): + evidence = {"url": response.url, "path": path, "status": response.status, + "content_type": response.headers.get("content-type", ""), "authentication_barrier": "not observed"} + hits.append(evidence) + result.findings.append(Finding(context.run_id, target.url, self.name, "unauthenticated-service", + f"Potential unauthenticated surface at {path}", "high" if SENSITIVE.search(path) else "medium", .85, + "A bounded GET request reached a potentially sensitive interface without an observed authentication barrier.", + evidence, ["authorization", "native", "noauth"], + remediation="Require server-side authentication and authorization before returning sensitive interface content.")) + context.emit(target.url, self.name, "finding", f"Potential unauthenticated surface: {response.url}") + if context.rate > 0: time.sleep(min(1.0 / context.rate, 1.0)) + result.observations = {"engine": "native-redflare", "paths_checked": len(PATHS), "candidate_surfaces": len(hits)} + directory = context.artifact_dir / self.name / target.host; directory.mkdir(parents=True, exist_ok=True) + artifact = directory / "findings.json"; artifact.write_text(json.dumps({"target": target.url, "findings": hits}, indent=2), encoding="utf-8") + result.artifacts.append(str(artifact)); result.duration_seconds = round(time.monotonic() - started, 4); return result diff --git a/redflare/modules/repository.py b/redflare/modules/repository.py new file mode 100644 index 0000000..7cf937d --- /dev/null +++ b/redflare/modules/repository.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import base64 +import json +import os +import urllib.parse +import urllib.request +from pathlib import Path + +from .exposure import SensitiveExposureModule + +SKIP_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".gz", ".woff", ".ico", ".lock") + + +def normalize_repository(value: str) -> str: + value = value.strip().removesuffix(".git") + parsed = urllib.parse.urlsplit(value if "://" in value else "https://github.com/" + value) + parts = [part for part in parsed.path.split("/") if part] + if parsed.hostname != "github.com" or len(parts) != 2: + raise ValueError(f"invalid GitHub repository {value!r}; expected owner/repository or GitHub URL") + return f"{parts[0]}/{parts[1]}" + + +def _github(path: str, token: str, timeout: int) -> dict: + headers = {"Accept": "application/vnd.github+json", "User-Agent": "REDflare-v2/native-repository-intelligence", + "X-GitHub-Api-Version": "2022-11-28", "Authorization": f"Bearer {token}"} + with urllib.request.urlopen(urllib.request.Request("https://api.github.com" + path, headers=headers), timeout=timeout) as response: + return json.load(response) + + +def run_repository_intelligence(repositories: list[str], output: Path, timeout: int = 60) -> dict: + token = os.environ.get("GITHUB_TOKEN", "").strip() + if not token: + return {"status": "error", "error": "GITHUB_TOKEN is required for native repository intelligence"} + output.mkdir(parents=True, exist_ok=True); findings = []; scanned = 0; errors = [] + for raw in repositories: + try: + repo = normalize_repository(raw); metadata = _github(f"/repos/{repo}", token, timeout) + branch = metadata.get("default_branch") or "main" + tree = _github(f"/repos/{repo}/git/trees/{urllib.parse.quote(branch, safe='')}?recursive=1", token, timeout) + if tree.get("truncated"): raise RuntimeError("repository tree truncated; refusing incomplete scan") + for entry in tree.get("tree", []): + path = str(entry.get("path") or "") + if entry.get("type") != "blob" or path.lower().endswith(SKIP_SUFFIXES) or int(entry.get("size") or 0) > 1_000_000: continue + blob = _github(f"/repos/{repo}/git/blobs/{entry['sha']}", token, timeout) + if blob.get("encoding") != "base64": continue + text = base64.b64decode(blob.get("content", "")).decode("utf-8", errors="replace"); scanned += 1 + url = f"https://github.com/{repo}/blob/{branch}/{path}" + for evidence in SensitiveExposureModule.detect(text, url): + findings.append({"repository": repo, "branch": branch, "path": path, **evidence}) + except Exception as exc: + errors.append(f"{raw}: {type(exc).__name__}: {exc}") + artifact = output / "repository_findings.json" + artifact.write_text(json.dumps({"findings": findings, "values_masked": True}, indent=2), encoding="utf-8") + return {"status": "completed" if not errors else "error", "engine": "native-redflare", "repositories": len(repositories), + "files_scanned": scanned, "findings": len(findings), "errors": errors, "output": str(artifact)} diff --git a/redflare/modules/vulnerabilities.py b/redflare/modules/vulnerabilities.py new file mode 100644 index 0000000..a7f7575 --- /dev/null +++ b/redflare/modules/vulnerabilities.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import json +import os +import re +import threading +import time +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + +from redflare.core.models import Finding, ModuleResult, Target + +from .base import Module, ModuleContext +from .http import request + + +NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0" +NVD_DETAIL = "https://nvd.nist.gov/vuln/detail/" +CVE_DETAIL = "https://www.cve.org/CVERecord?id=" +_NVD_LOCK = threading.Lock() +_LAST_NVD_REQUEST = 0.0 + + +@dataclass(frozen=True) +class Fingerprint: + product: str + version: str + vendor: str + cpe_product: str + source: str + evidence: str + + @property + def cpe(self) -> str: + version = urllib.parse.quote(self.version, safe="._-") + return f"cpe:2.3:a:{self.vendor}:{self.cpe_product}:{version}:*:*:*:*:*:*:*" + + +PRODUCTS = { + "apache": ("Apache HTTP Server", "apache", "http_server"), + "nginx": ("nginx", "f5", "nginx"), + "openresty": ("OpenResty", "openresty", "openresty"), + "microsoft-iis": ("Microsoft IIS", "microsoft", "internet_information_services"), + "php": ("PHP", "php", "php"), + "jquery": ("jQuery", "jquery", "jquery"), + "bootstrap": ("Bootstrap", "getbootstrap", "bootstrap"), + "wordpress": ("WordPress", "wordpress", "wordpress"), + "drupal": ("Drupal", "drupal", "drupal"), + "joomla": ("Joomla", "joomla", r"joomla\!"), +} + +HEADER_PATTERNS = ( + ("server", re.compile(r"\b(Apache|nginx|openresty|Microsoft-IIS)/([0-9][0-9A-Za-z._-]*)", re.I)), + ("x-powered-by", re.compile(r"\b(PHP)/([0-9][0-9A-Za-z._-]*)", re.I)), +) +BODY_PATTERNS = ( + ("html-generator", re.compile(r']+name=["\']generator["\'][^>]+content=["\']\s*(WordPress|Drupal|Joomla!?)\s+([0-9][0-9A-Za-z._-]*)', re.I)), + ("html-generator", re.compile(r']+content=["\']\s*(WordPress|Drupal|Joomla!?)\s+([0-9][0-9A-Za-z._-]*)["\'][^>]+name=["\']generator["\']', re.I)), + ("asset-url", re.compile(r"(?:jquery)[-.]([0-9]+\.[0-9]+(?:\.[0-9]+)?)(?:\.min)?\.js", re.I)), + ("asset-url", re.compile(r"(?:bootstrap)[-.]([0-9]+\.[0-9]+(?:\.[0-9]+)?)(?:\.min)?\.(?:js|css)", re.I)), +) + + +def fingerprint_response(headers: dict[str, str], body: bytes) -> list[Fingerprint]: + found: dict[tuple[str, str], Fingerprint] = {} + for header, pattern in HEADER_PATTERNS: + value = headers.get(header, "") + for match in pattern.finditer(value): + key = match.group(1).lower() + _add_fingerprint(found, key, match.group(2), f"header:{header}", match.group(0)) + + text = body.decode("utf-8", errors="replace") + for source, pattern in BODY_PATTERNS: + for match in pattern.finditer(text): + if pattern.groups == 2: + key, version = match.group(1).lower().rstrip("!"), match.group(2) + else: + key = "jquery" if "jquery" in match.group(0).lower() else "bootstrap" + version = match.group(1) + _add_fingerprint(found, key, version, source, match.group(0)[:180]) + return sorted(found.values(), key=lambda item: (item.product.lower(), item.version)) + + +def _add_fingerprint(found: dict[tuple[str, str], Fingerprint], key: str, version: str, source: str, evidence: str) -> None: + definition = PRODUCTS.get(key) + if not definition or not re.fullmatch(r"[0-9][0-9A-Za-z._-]{0,63}", version): + return + product, vendor, cpe_product = definition + found.setdefault((product, version), Fingerprint(product, version, vendor, cpe_product, source, evidence)) + + +def query_nvd(fingerprint: Fingerprint, timeout: float, limit: int) -> list[dict[str, Any]]: + query = urllib.parse.urlencode({"virtualMatchString": fingerprint.cpe, "resultsPerPage": min(limit, 100)} ) + headers = {"User-Agent": "REDflare-v2/2.1 authorized-assessment"} + api_key = os.environ.get("NVD_API_KEY", "").strip() + if api_key: + headers["apiKey"] = api_key + req = urllib.request.Request(f"{NVD_API}?{query}", headers=headers) + global _LAST_NVD_REQUEST + with _NVD_LOCK: + delay = 0.7 if api_key else 6.1 + wait = delay - (time.monotonic() - _LAST_NVD_REQUEST) + if wait > 0: + time.sleep(wait) + with urllib.request.urlopen(req, timeout=timeout) as response: + payload = json.load(response) + _LAST_NVD_REQUEST = time.monotonic() + return [item.get("cve", {}) for item in payload.get("vulnerabilities", [])[:limit] if item.get("cve")] + + +def _english_description(cve: dict[str, Any]) -> str: + descriptions = cve.get("descriptions") or [] + return next((item.get("value", "") for item in descriptions if item.get("lang") == "en"), "No English NVD description available.") + + +def _cvss(cve: dict[str, Any]) -> tuple[float | None, str, str]: + metrics = cve.get("metrics") or {} + for family in ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2"): + values = metrics.get(family) or [] + if not values: + continue + data = values[0].get("cvssData") or {} + score = data.get("baseScore") + severity = str(data.get("baseSeverity") or values[0].get("baseSeverity") or "unknown").lower() + return (float(score) if score is not None else None, severity, str(data.get("vectorString") or "")) + return None, "unknown", "" + + +def _severity(value: str, score: float | None) -> str: + if value in {"critical", "high", "medium", "low"}: + return value + if score is None: + return "info" + return "critical" if score >= 9 else "high" if score >= 7 else "medium" if score >= 4 else "low" + + +def _references(cve: dict[str, Any], cve_id: str) -> list[str]: + preferred = [str(item.get("url")) for item in cve.get("references") or [] if item.get("url")] + return list(dict.fromkeys([f"{NVD_DETAIL}{cve_id}", f"{CVE_DETAIL}{cve_id}", *preferred]))[:12] + + +class CVEIntelligenceModule(Module): + name = "cve_intelligence" + description = "Fingerprint disclosed component versions and correlate exact CPEs with NVD CVE records" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic() + result = ModuleResult(self.name, target.url) + try: + context.emit(target.url, self.name, "progress", "Fingerprinting disclosed component versions") + response = request(target.url, context.timeout, method="GET", max_body=1_000_000) + fingerprints = fingerprint_response(response.headers, response.body) + mapped_urls = context.surface_graph.request_urls(target.url, "GET") + fingerprints.extend(fingerprint_response({}, "\n".join(mapped_urls).encode())) + fingerprints = list({(item.product, item.version): item for item in fingerprints}.values()) + fingerprints.sort(key=lambda item: (item.product.lower(), item.version)) + fingerprints = fingerprints[: context.max_cve_products] + result.observations["fingerprints"] = [item.__dict__ | {"cpe": item.cpe} for item in fingerprints] + result.observations["source"] = "NVD CVE API 2.0" + if not fingerprints: + context.emit(target.url, self.name, "info", "No exact component versions were disclosed; CVE correlation skipped") + for fingerprint in fingerprints: + context.emit(target.url, self.name, "progress", f"Checking {fingerprint.product} {fingerprint.version} against NVD") + try: + records = query_nvd(fingerprint, max(context.timeout, 15), context.max_cves_per_product) + except Exception as exc: + result.errors.append(f"NVD lookup failed for {fingerprint.product} {fingerprint.version}: {type(exc).__name__}: {exc}") + context.emit(target.url, self.name, "error", result.errors[-1]) + continue + for cve in records: + cve_id = str(cve.get("id") or "") + if not re.fullmatch(r"CVE-\d{4}-\d{4,}", cve_id) or str(cve.get("vulnStatus")).lower() == "rejected": + continue + score, cvss_severity, vector = _cvss(cve) + severity = _severity(cvss_severity, score) + known_exploited = bool(cve.get("cisaExploitAdd")) + if known_exploited and severity in {"info", "low", "medium"}: + severity = "high" + links = _references(cve, cve_id) + evidence = { + "cve": cve_id, + "product": fingerprint.product, + "version": fingerprint.version, + "detected_by": fingerprint.source, + "fingerprint_evidence": fingerprint.evidence, + "matched_cpe": fingerprint.cpe, + "cvss_score": score, + "cvss_vector": vector, + "published": cve.get("published"), + "last_modified": cve.get("lastModified"), + "references": links, + "nvd_status": cve.get("vulnStatus"), + "cisa_known_exploited": known_exploited, + "cisa_kev_added": cve.get("cisaExploitAdd"), + "cisa_action_due": cve.get("cisaActionDue"), + "cisa_required_action": cve.get("cisaRequiredAction"), + } + finding = Finding( + context.run_id, target.url, self.name, "known-vulnerable-component", + f"{cve_id} affects disclosed {fingerprint.product} {fingerprint.version}", + severity, 0.9, _english_description(cve), evidence, + ["cve", "component", "known-vulnerability", fingerprint.product.lower().replace(" ", "-")], + remediation="Confirm the component inventory, review the vendor advisory, and upgrade to a non-affected supported release.", + ) + finding.standards["CVE"] = [{"id": cve_id, "url": links[0], "version": str(cve.get("published") or "")[:4]}] + result.findings.append(finding) + context.emit(target.url, self.name, "finding", f"{cve_id} | {severity.upper()} | {fingerprint.product} {fingerprint.version}") + result.observations["cves_found"] = len(result.findings) + except Exception as exc: + result.status = "error" + result.errors.append(f"{type(exc).__name__}: {exc}") + result.duration_seconds = round(time.monotonic() - started, 4) + return result