Upload files to "redflare/modules"

redflare/modules/__init__.py

@@ -0,0 +1,21 @@
+from .headers import HeaderModule
+from .exposure import SensitiveExposureModule
+from .mapping import ApplicationMappingModule
+from .passive import PassiveReconModule
+from .paths import PathDiscoveryModule
+from .surface import SurfaceAnalysisModule
+from .vulnerabilities import CVEIntelligenceModule
+from .browser import NativeBrowserRuntimeModule
+from .noauth import NativeNoAuthModule
+
+__all__ = [
+    "ApplicationMappingModule",
+    "HeaderModule",
+    "PassiveReconModule",
+    "PathDiscoveryModule",
+    "SurfaceAnalysisModule",
+    "SensitiveExposureModule",
+    "CVEIntelligenceModule",
+    "NativeBrowserRuntimeModule",
+    "NativeNoAuthModule",
+]

redflare/modules/browser.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from urllib.parse import urlsplit
+
+from redflare.core.models import Finding, ModuleResult, Target
+from .base import Module, ModuleContext
+from .headers import SECURITY_HEADERS
+from .http import request
+
+INTERESTING = ("api", "graphql", "admin", "auth", "token", "upload", "config", "internal")
+
+
+class NativeBrowserRuntimeModule(Module):
+    name = "browser_runtime"
+    description = "Natively capture browser/runtime requests, responses, redirects, console events, and application endpoints"
+
+    def run(self, target: Target, context: ModuleContext) -> ModuleResult:
+        started = time.monotonic(); result = ModuleResult(self.name, target.url)
+        try:
+            capture = asyncio.run(self._capture(target, context))
+        except Exception as exc:
+            capture = self._fallback(target, context, f"{type(exc).__name__}: {exc}")
+        requests_seen = capture.get("requests", []); responses = capture.get("responses", [])
+        for item in requests_seen:
+            url = str(item.get("url") or ""); parsed = urlsplit(url)
+            if parsed.hostname == target.host:
+                context.surface_graph.add_endpoint(target.url, url, method=str(item.get("method") or "GET"), source=self.name,
+                                                   authentication="observed" if item.get("authentication") else None)
+        headers = {k.lower(): v for k, v in (capture.get("main_headers") or {}).items()}
+        missing = [name for name in SECURITY_HEADERS if name not in headers]
+        if missing:
+            result.findings.append(Finding(context.run_id, target.url, self.name, "browser-security-headers",
+                "Browser-runtime response headers need hardening", "low", .95,
+                "Native runtime capture confirmed missing common security headers.", {"missing": missing}, ["browser", "headers", "native"]))
+        interesting = [item for item in requests_seen if any(term in str(item.get("url", "")).lower() for term in INTERESTING)]
+        if interesting:
+            result.findings.append(Finding(context.run_id, target.url, self.name, "browser-endpoints",
+                "Interesting endpoints observed during browser execution", "info", .8,
+                "Native runtime capture observed assessment-relevant endpoints.", {"endpoints": interesting[:50]}, ["browser", "endpoints", "native"]))
+        result.observations = {"engine": capture.get("engine"), "requests": len(requests_seen), "responses": len(responses),
+                               "console_events": len(capture.get("console", [])), "final_url": capture.get("final_url"),
+                               "fallback_reason": capture.get("fallback_reason")}
+        directory = context.artifact_dir / self.name / target.host; directory.mkdir(parents=True, exist_ok=True)
+        artifact = directory / "network_capture.json"; artifact.write_text(json.dumps(capture, indent=2), encoding="utf-8")
+        result.artifacts.append(str(artifact)); result.duration_seconds = round(time.monotonic() - started, 4); return result
+
+    async def _capture(self, target: Target, context: ModuleContext) -> dict:
+        from playwright.async_api import async_playwright
+        captured = {"engine": "native-playwright", "target": target.url, "requests": [], "responses": [], "console": [], "main_headers": {}}
+        async with async_playwright() as runtime:
+            browser = await runtime.chromium.launch(headless=True)
+            browser_context = await browser.new_context(ignore_https_errors=True)
+            page = await browser_context.new_page()
+            page.on("request", lambda req: captured["requests"].append({"url": req.url, "method": req.method, "resource_type": req.resource_type}))
+            page.on("response", lambda res: captured["responses"].append({"url": res.url, "status": res.status}))
+            page.on("console", lambda msg: captured["console"].append({"type": msg.type, "text": msg.text[:500]}))
+            response = await page.goto(target.url, wait_until="domcontentloaded", timeout=int(max(5, context.timeout) * 1000))
+            if response: captured["main_headers"] = await response.all_headers()
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(1500)
+            captured["final_url"] = page.url; captured["title"] = await page.title()
+            await page.screenshot(path=str(context.artifact_dir / f"{target.host}_runtime.png"), full_page=True)
+            await browser.close()
+        return captured
+
+    @staticmethod
+    def _fallback(target: Target, context: ModuleContext, reason: str) -> dict:
+        response = request(target.url, context.timeout, max_body=500_000)
+        urls = context.surface_graph.request_urls(target.url)
+        return {"engine": "native-http-fallback", "fallback_reason": reason, "target": target.url,
+                "final_url": response.url, "main_headers": response.headers,
+                "requests": [{"url": url, "method": "GET", "resource_type": "mapped"} for url in urls] or [{"url": response.url, "method": "GET", "resource_type": "document"}],
+                "responses": [{"url": response.url, "status": response.status}], "console": []}

redflare/modules/noauth.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import json
+import re
+import time
+from urllib.parse import urljoin
+
+from redflare.core.models import Finding, ModuleResult, Target
+from .base import Module, ModuleContext
+from .http import request
+
+PATHS = ("/", "/login", "/admin", "/dashboard", "/panel", "/console", "/manage", "/manager",
+         "/status", "/api/status", "/health", "/metrics", "/config", "/config.json", "/api/config",
+         "/graphql", "/swagger-ui.html", "/openapi.json", "/actuator/env", "/debug/vars", "/.env")
+SENSITIVE = re.compile(r"(?:config|secret|credential|token|\.env|actuator|debug|metrics|console|admin|dashboard)", re.I)
+AUTH = re.compile(r"(?:type=[\"']?password|/(?:login|signin|auth)(?:/|\?|$)|sign\s*in|log\s*in)", re.I)
+
+
+class NativeNoAuthModule(Module):
+    name = "authorization_surface"
+    description = "Natively triage bounded in-scope web surfaces for missing authentication barriers"
+
+    def run(self, target: Target, context: ModuleContext) -> ModuleResult:
+        started = time.monotonic(); result = ModuleResult(self.name, target.url); hits = []
+        for path in PATHS:
+            url = urljoin(target.url.rstrip("/") + "/", path.lstrip("/"))
+            try:
+                response = request(url, context.timeout, max_body=200_000, allowed_origin=(target.host, target.port))
+            except Exception:
+                continue
+            body = response.body.decode("utf-8", errors="replace")
+            authenticated = response.status in {401, 403} or "www-authenticate" in response.headers or bool(AUTH.search(body))
+            context.surface_graph.add_endpoint(target.url, response.url, method="GET", source=self.name,
+                                               content_type=response.headers.get("content-type"), status=response.status,
+                                               authentication="required" if authenticated else "not-observed")
+            if response.status == 200 and not authenticated and (path != "/" or SENSITIVE.search(body)) and SENSITIVE.search(path + " " + body[:5000]):
+                evidence = {"url": response.url, "path": path, "status": response.status,
+                            "content_type": response.headers.get("content-type", ""), "authentication_barrier": "not observed"}
+                hits.append(evidence)
+                result.findings.append(Finding(context.run_id, target.url, self.name, "unauthenticated-service",
+                    f"Potential unauthenticated surface at {path}", "high" if SENSITIVE.search(path) else "medium", .85,
+                    "A bounded GET request reached a potentially sensitive interface without an observed authentication barrier.",
+                    evidence, ["authorization", "native", "noauth"],
+                    remediation="Require server-side authentication and authorization before returning sensitive interface content."))
+                context.emit(target.url, self.name, "finding", f"Potential unauthenticated surface: {response.url}")
+            if context.rate > 0: time.sleep(min(1.0 / context.rate, 1.0))
+        result.observations = {"engine": "native-redflare", "paths_checked": len(PATHS), "candidate_surfaces": len(hits)}
+        directory = context.artifact_dir / self.name / target.host; directory.mkdir(parents=True, exist_ok=True)
+        artifact = directory / "findings.json"; artifact.write_text(json.dumps({"target": target.url, "findings": hits}, indent=2), encoding="utf-8")
+        result.artifacts.append(str(artifact)); result.duration_seconds = round(time.monotonic() - started, 4); return result

redflare/modules/repository.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+from .exposure import SensitiveExposureModule
+
+SKIP_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".gz", ".woff", ".ico", ".lock")
+
+
+def normalize_repository(value: str) -> str:
+    value = value.strip().removesuffix(".git")
+    parsed = urllib.parse.urlsplit(value if "://" in value else "https://github.com/" + value)
+    parts = [part for part in parsed.path.split("/") if part]
+    if parsed.hostname != "github.com" or len(parts) != 2:
+        raise ValueError(f"invalid GitHub repository {value!r}; expected owner/repository or GitHub URL")
+    return f"{parts[0]}/{parts[1]}"
+
+
+def _github(path: str, token: str, timeout: int) -> dict:
+    headers = {"Accept": "application/vnd.github+json", "User-Agent": "REDflare-v2/native-repository-intelligence",
+               "X-GitHub-Api-Version": "2022-11-28", "Authorization": f"Bearer {token}"}
+    with urllib.request.urlopen(urllib.request.Request("https://api.github.com" + path, headers=headers), timeout=timeout) as response:
+        return json.load(response)
+
+
+def run_repository_intelligence(repositories: list[str], output: Path, timeout: int = 60) -> dict:
+    token = os.environ.get("GITHUB_TOKEN", "").strip()
+    if not token:
+        return {"status": "error", "error": "GITHUB_TOKEN is required for native repository intelligence"}
+    output.mkdir(parents=True, exist_ok=True); findings = []; scanned = 0; errors = []
+    for raw in repositories:
+        try:
+            repo = normalize_repository(raw); metadata = _github(f"/repos/{repo}", token, timeout)
+            branch = metadata.get("default_branch") or "main"
+            tree = _github(f"/repos/{repo}/git/trees/{urllib.parse.quote(branch, safe='')}?recursive=1", token, timeout)
+            if tree.get("truncated"): raise RuntimeError("repository tree truncated; refusing incomplete scan")
+            for entry in tree.get("tree", []):
+                path = str(entry.get("path") or "")
+                if entry.get("type") != "blob" or path.lower().endswith(SKIP_SUFFIXES) or int(entry.get("size") or 0) > 1_000_000: continue
+                blob = _github(f"/repos/{repo}/git/blobs/{entry['sha']}", token, timeout)
+                if blob.get("encoding") != "base64": continue
+                text = base64.b64decode(blob.get("content", "")).decode("utf-8", errors="replace"); scanned += 1
+                url = f"https://github.com/{repo}/blob/{branch}/{path}"
+                for evidence in SensitiveExposureModule.detect(text, url):
+                    findings.append({"repository": repo, "branch": branch, "path": path, **evidence})
+        except Exception as exc:
+            errors.append(f"{raw}: {type(exc).__name__}: {exc}")
+    artifact = output / "repository_findings.json"
+    artifact.write_text(json.dumps({"findings": findings, "values_masked": True}, indent=2), encoding="utf-8")
+    return {"status": "completed" if not errors else "error", "engine": "native-redflare", "repositories": len(repositories),
+            "files_scanned": scanned, "findings": len(findings), "errors": errors, "output": str(artifact)}

redflare/modules/vulnerabilities.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Any
+
+from redflare.core.models import Finding, ModuleResult, Target
+
+from .base import Module, ModuleContext
+from .http import request
+
+
+NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+NVD_DETAIL = "https://nvd.nist.gov/vuln/detail/"
+CVE_DETAIL = "https://www.cve.org/CVERecord?id="
+_NVD_LOCK = threading.Lock()
+_LAST_NVD_REQUEST = 0.0
+
+
+@dataclass(frozen=True)
+class Fingerprint:
+    product: str
+    version: str
+    vendor: str
+    cpe_product: str
+    source: str
+    evidence: str
+
+    @property
+    def cpe(self) -> str:
+        version = urllib.parse.quote(self.version, safe="._-")
+        return f"cpe:2.3:a:{self.vendor}:{self.cpe_product}:{version}:*:*:*:*:*:*:*"
+
+
+PRODUCTS = {
+    "apache": ("Apache HTTP Server", "apache", "http_server"),
+    "nginx": ("nginx", "f5", "nginx"),
+    "openresty": ("OpenResty", "openresty", "openresty"),
+    "microsoft-iis": ("Microsoft IIS", "microsoft", "internet_information_services"),
+    "php": ("PHP", "php", "php"),
+    "jquery": ("jQuery", "jquery", "jquery"),
+    "bootstrap": ("Bootstrap", "getbootstrap", "bootstrap"),
+    "wordpress": ("WordPress", "wordpress", "wordpress"),
+    "drupal": ("Drupal", "drupal", "drupal"),
+    "joomla": ("Joomla", "joomla", r"joomla\!"),
+}
+
+HEADER_PATTERNS = (
+    ("server", re.compile(r"\b(Apache|nginx|openresty|Microsoft-IIS)/([0-9][0-9A-Za-z._-]*)", re.I)),
+    ("x-powered-by", re.compile(r"\b(PHP)/([0-9][0-9A-Za-z._-]*)", re.I)),
+)
+BODY_PATTERNS = (
+    ("html-generator", re.compile(r'<meta[^>]+name=["\']generator["\'][^>]+content=["\']\s*(WordPress|Drupal|Joomla!?)\s+([0-9][0-9A-Za-z._-]*)', re.I)),
+    ("html-generator", re.compile(r'<meta[^>]+content=["\']\s*(WordPress|Drupal|Joomla!?)\s+([0-9][0-9A-Za-z._-]*)["\'][^>]+name=["\']generator["\']', re.I)),
+    ("asset-url", re.compile(r"(?:jquery)[-.]([0-9]+\.[0-9]+(?:\.[0-9]+)?)(?:\.min)?\.js", re.I)),
+    ("asset-url", re.compile(r"(?:bootstrap)[-.]([0-9]+\.[0-9]+(?:\.[0-9]+)?)(?:\.min)?\.(?:js|css)", re.I)),
+)
+
+
+def fingerprint_response(headers: dict[str, str], body: bytes) -> list[Fingerprint]:
+    found: dict[tuple[str, str], Fingerprint] = {}
+    for header, pattern in HEADER_PATTERNS:
+        value = headers.get(header, "")
+        for match in pattern.finditer(value):
+            key = match.group(1).lower()
+            _add_fingerprint(found, key, match.group(2), f"header:{header}", match.group(0))
+
+    text = body.decode("utf-8", errors="replace")
+    for source, pattern in BODY_PATTERNS:
+        for match in pattern.finditer(text):
+            if pattern.groups == 2:
+                key, version = match.group(1).lower().rstrip("!"), match.group(2)
+            else:
+                key = "jquery" if "jquery" in match.group(0).lower() else "bootstrap"
+                version = match.group(1)
+            _add_fingerprint(found, key, version, source, match.group(0)[:180])
+    return sorted(found.values(), key=lambda item: (item.product.lower(), item.version))
+
+
+def _add_fingerprint(found: dict[tuple[str, str], Fingerprint], key: str, version: str, source: str, evidence: str) -> None:
+    definition = PRODUCTS.get(key)
+    if not definition or not re.fullmatch(r"[0-9][0-9A-Za-z._-]{0,63}", version):
+        return
+    product, vendor, cpe_product = definition
+    found.setdefault((product, version), Fingerprint(product, version, vendor, cpe_product, source, evidence))
+
+
+def query_nvd(fingerprint: Fingerprint, timeout: float, limit: int) -> list[dict[str, Any]]:
+    query = urllib.parse.urlencode({"virtualMatchString": fingerprint.cpe, "resultsPerPage": min(limit, 100)} )
+    headers = {"User-Agent": "REDflare-v2/2.1 authorized-assessment"}
+    api_key = os.environ.get("NVD_API_KEY", "").strip()
+    if api_key:
+        headers["apiKey"] = api_key
+    req = urllib.request.Request(f"{NVD_API}?{query}", headers=headers)
+    global _LAST_NVD_REQUEST
+    with _NVD_LOCK:
+        delay = 0.7 if api_key else 6.1
+        wait = delay - (time.monotonic() - _LAST_NVD_REQUEST)
+        if wait > 0:
+            time.sleep(wait)
+        with urllib.request.urlopen(req, timeout=timeout) as response:
+            payload = json.load(response)
+        _LAST_NVD_REQUEST = time.monotonic()
+    return [item.get("cve", {}) for item in payload.get("vulnerabilities", [])[:limit] if item.get("cve")]
+
+
+def _english_description(cve: dict[str, Any]) -> str:
+    descriptions = cve.get("descriptions") or []
+    return next((item.get("value", "") for item in descriptions if item.get("lang") == "en"), "No English NVD description available.")
+
+
+def _cvss(cve: dict[str, Any]) -> tuple[float | None, str, str]:
+    metrics = cve.get("metrics") or {}
+    for family in ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2"):
+        values = metrics.get(family) or []
+        if not values:
+            continue
+        data = values[0].get("cvssData") or {}
+        score = data.get("baseScore")
+        severity = str(data.get("baseSeverity") or values[0].get("baseSeverity") or "unknown").lower()
+        return (float(score) if score is not None else None, severity, str(data.get("vectorString") or ""))
+    return None, "unknown", ""
+
+
+def _severity(value: str, score: float | None) -> str:
+    if value in {"critical", "high", "medium", "low"}:
+        return value
+    if score is None:
+        return "info"
+    return "critical" if score >= 9 else "high" if score >= 7 else "medium" if score >= 4 else "low"
+
+
+def _references(cve: dict[str, Any], cve_id: str) -> list[str]:
+    preferred = [str(item.get("url")) for item in cve.get("references") or [] if item.get("url")]
+    return list(dict.fromkeys([f"{NVD_DETAIL}{cve_id}", f"{CVE_DETAIL}{cve_id}", *preferred]))[:12]
+
+
+class CVEIntelligenceModule(Module):
+    name = "cve_intelligence"
+    description = "Fingerprint disclosed component versions and correlate exact CPEs with NVD CVE records"
+
+    def run(self, target: Target, context: ModuleContext) -> ModuleResult:
+        started = time.monotonic()
+        result = ModuleResult(self.name, target.url)
+        try:
+            context.emit(target.url, self.name, "progress", "Fingerprinting disclosed component versions")
+            response = request(target.url, context.timeout, method="GET", max_body=1_000_000)
+            fingerprints = fingerprint_response(response.headers, response.body)
+            mapped_urls = context.surface_graph.request_urls(target.url, "GET")
+            fingerprints.extend(fingerprint_response({}, "\n".join(mapped_urls).encode()))
+            fingerprints = list({(item.product, item.version): item for item in fingerprints}.values())
+            fingerprints.sort(key=lambda item: (item.product.lower(), item.version))
+            fingerprints = fingerprints[: context.max_cve_products]
+            result.observations["fingerprints"] = [item.__dict__ | {"cpe": item.cpe} for item in fingerprints]
+            result.observations["source"] = "NVD CVE API 2.0"
+            if not fingerprints:
+                context.emit(target.url, self.name, "info", "No exact component versions were disclosed; CVE correlation skipped")
+            for fingerprint in fingerprints:
+                context.emit(target.url, self.name, "progress", f"Checking {fingerprint.product} {fingerprint.version} against NVD")
+                try:
+                    records = query_nvd(fingerprint, max(context.timeout, 15), context.max_cves_per_product)
+                except Exception as exc:
+                    result.errors.append(f"NVD lookup failed for {fingerprint.product} {fingerprint.version}: {type(exc).__name__}: {exc}")
+                    context.emit(target.url, self.name, "error", result.errors[-1])
+                    continue
+                for cve in records:
+                    cve_id = str(cve.get("id") or "")
+                    if not re.fullmatch(r"CVE-\d{4}-\d{4,}", cve_id) or str(cve.get("vulnStatus")).lower() == "rejected":
+                        continue
+                    score, cvss_severity, vector = _cvss(cve)
+                    severity = _severity(cvss_severity, score)
+                    known_exploited = bool(cve.get("cisaExploitAdd"))
+                    if known_exploited and severity in {"info", "low", "medium"}:
+                        severity = "high"
+                    links = _references(cve, cve_id)
+                    evidence = {
+                        "cve": cve_id,
+                        "product": fingerprint.product,
+                        "version": fingerprint.version,
+                        "detected_by": fingerprint.source,
+                        "fingerprint_evidence": fingerprint.evidence,
+                        "matched_cpe": fingerprint.cpe,
+                        "cvss_score": score,
+                        "cvss_vector": vector,
+                        "published": cve.get("published"),
+                        "last_modified": cve.get("lastModified"),
+                        "references": links,
+                        "nvd_status": cve.get("vulnStatus"),
+                        "cisa_known_exploited": known_exploited,
+                        "cisa_kev_added": cve.get("cisaExploitAdd"),
+                        "cisa_action_due": cve.get("cisaActionDue"),
+                        "cisa_required_action": cve.get("cisaRequiredAction"),
+                    }
+                    finding = Finding(
+                        context.run_id, target.url, self.name, "known-vulnerable-component",
+                        f"{cve_id} affects disclosed {fingerprint.product} {fingerprint.version}",
+                        severity, 0.9, _english_description(cve), evidence,
+                        ["cve", "component", "known-vulnerability", fingerprint.product.lower().replace(" ", "-")],
+                        remediation="Confirm the component inventory, review the vendor advisory, and upgrade to a non-affected supported release.",
+                    )
+                    finding.standards["CVE"] = [{"id": cve_id, "url": links[0], "version": str(cve.get("published") or "")[:4]}]
+                    result.findings.append(finding)
+                    context.emit(target.url, self.name, "finding", f"{cve_id} | {severity.upper()} | {fingerprint.product} {fingerprint.version}")
+            result.observations["cves_found"] = len(result.findings)
+        except Exception as exc:
+            result.status = "error"
+            result.errors.append(f"{type(exc).__name__}: {exc}")
+        result.duration_seconds = round(time.monotonic() - started, 4)
+        return result