diff --git a/redflare/modules/headers.py b/redflare/modules/headers.py new file mode 100644 index 0000000..2a604a6 --- /dev/null +++ b/redflare/modules/headers.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import time + +from redflare.core.models import Finding, ModuleResult, Target +from .base import Module, ModuleContext +from .http import request + + +SECURITY_HEADERS = { + "strict-transport-security": "HSTS", + "content-security-policy": "Content Security Policy", + "x-content-type-options": "MIME sniffing protection", + "referrer-policy": "Referrer Policy", + "permissions-policy": "Permissions Policy", +} + + +class HeaderModule(Module): + name = "http_headers" + description = "Inspect HTTP status, redirects, server metadata, and security headers" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic() + result = ModuleResult(self.name, target.url) + try: + context.emit(target.url, self.name, "progress", f"Requesting {target.url}") + response = request(target.url, context.timeout, method="GET", max_body=200_000) + missing = [header for header in SECURITY_HEADERS if header not in response.headers] + result.observations.update( + { + "status": response.status, + "final_url": response.url, + "headers": response.headers, + "body_bytes_sampled": len(response.body), + "missing_security_headers": missing, + } + ) + context.emit( + target.url, + self.name, + "info", + f"HTTP {response.status}; final={response.url}; sampled={len(response.body)} bytes", + ) + if missing: + result.findings.append( + Finding( + context.run_id, + target.url, + self.name, + "security-headers", + "Common security headers are missing", + "low", + 0.95, + "The response omitted: " + ", ".join(missing), + {"status": response.status, "missing": missing}, + ["headers", "hardening"], + ) + ) + except Exception as exc: + result.status = "error" + result.errors.append(f"{type(exc).__name__}: {exc}") + result.duration_seconds = round(time.monotonic() - started, 4) + return result diff --git a/redflare/modules/http.py b/redflare/modules/http.py new file mode 100644 index 0000000..efd8aaa --- /dev/null +++ b/redflare/modules/http.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import ssl +import urllib.error +import urllib.request +from dataclasses import dataclass +from urllib.parse import urlparse + + +@dataclass +class HTTPResponse: + url: str + status: int + headers: dict[str, str] + body: bytes + + +class ScopedRedirectHandler(urllib.request.HTTPRedirectHandler): + def __init__(self, allowed_origin: tuple[str, int]): + super().__init__() + self.allowed_origin = allowed_origin + + def redirect_request(self, req, fp, code, msg, headers, newurl): + parsed = urlparse(newurl) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if (parsed.hostname, port) != self.allowed_origin: + return None + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def request( + url: str, + timeout: float, + method: str = "GET", + max_body: int = 1_000_000, + *, + data: bytes | None = None, + headers: dict[str, str] | None = None, + allowed_origin: tuple[str, int] | None = None, +) -> HTTPResponse: + request_headers = {"User-Agent": "REDflare-v2/2.1 authorized-assessment"} + request_headers.update(headers or {}) + req = urllib.request.Request(url, data=data, method=method, headers=request_headers) + context = ssl.create_default_context() + try: + if allowed_origin: + opener = urllib.request.build_opener( + ScopedRedirectHandler(allowed_origin), urllib.request.HTTPSHandler(context=context) + ) + response = opener.open(req, timeout=timeout) + else: + response = urllib.request.urlopen(req, timeout=timeout, context=context) + except urllib.error.HTTPError as exc: + response = exc + with response: + body = b"" if method == "HEAD" else response.read(max_body) + return HTTPResponse( + url=response.geturl(), + status=int(response.status), + headers={key.lower(): value for key, value in response.headers.items()}, + body=body, + ) diff --git a/redflare/modules/mapping.py b/redflare/modules/mapping.py new file mode 100644 index 0000000..3418047 --- /dev/null +++ b/redflare/modules/mapping.py @@ -0,0 +1,407 @@ +from __future__ import annotations + +import json +import re +import time +from collections import deque +from html.parser import HTMLParser +from typing import Any +from urllib.parse import urljoin, urlparse + +from redflare.core.models import Finding, ModuleResult, Target +from .base import Module, ModuleContext +from .http import request + + +OPENAPI_PATHS = ( + "/openapi.json", + "/swagger.json", + "/api/openapi.json", + "/api/swagger.json", + "/v3/api-docs", +) +HTTP_METHODS = {"get", "post", "put", "patch", "delete", "options", "head", "trace"} +ROUTE_LITERAL = re.compile(r"[\"']((?:https?://[^\"']+)|(?:/[A-Za-z0-9_~!$&'()*+,;=:@%./?{}-]{2,}))['\"]") +FETCH_CALL = re.compile(r"fetch\s*\(\s*[\"']([^\"']+)[\"']", re.I) +AXIOS_CALL = re.compile(r"axios\.(get|post|put|patch|delete)\s*\(\s*[\"']([^\"']+)[\"']", re.I) + + +class MappingParser(HTMLParser): + def __init__(self, base_url: str): + super().__init__() + self.base_url = base_url + self.links: list[str] = [] + self.scripts: list[str] = [] + self.forms: list[dict[str, Any]] = [] + self._form: dict[str, Any] | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = dict(attrs) + if tag in {"a", "area"} and values.get("href"): + self.links.append(urljoin(self.base_url, values["href"] or "")) + elif tag == "script" and values.get("src"): + self.scripts.append(urljoin(self.base_url, values["src"] or "")) + elif tag == "form": + self._form = { + "url": urljoin(self.base_url, values.get("action") or self.base_url), + "method": (values.get("method") or "GET").upper(), + "content_type": values.get("enctype") or "application/x-www-form-urlencoded", + "parameters": [], + } + self.forms.append(self._form) + elif tag in {"input", "select", "textarea", "button"} and self._form is not None: + name = values.get("name") + if name: + self._form["parameters"].append( + { + "name": name, + "location": "query" if self._form["method"] == "GET" else "body", + "required": "required" in values, + "data_type": values.get("type") or tag, + } + ) + + def handle_endtag(self, tag: str) -> None: + if tag == "form": + self._form = None + + +class ApplicationMappingModule(Module): + name = "application_mapping" + description = "Build a shared endpoint graph from pages, forms, JavaScript, API schemas, and optional GraphQL metadata" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic() + result = ModuleResult(self.name, target.url) + pages_visited: list[str] = [] + scripts_seen: set[str] = set() + schema_candidates = {urljoin(target.url.rstrip("/") + "/", path.lstrip("/")) for path in OPENAPI_PATHS} + queue = deque([(target.url, 0, "seed")]) + queued = {target.url} + + try: + while queue and len(pages_visited) < context.max_crawl_pages: + page_url, depth, source = queue.popleft() + if not self._same_origin(target, page_url): + continue + context.emit(target.url, self.name, "progress", f"Mapping {page_url}") + try: + response = request( + page_url, + context.timeout, + max_body=1_000_000, + allowed_origin=(target.host, target.port), + ) + except Exception as exc: + result.errors.append(f"{page_url}: {type(exc).__name__}: {exc}") + continue + pages_visited.append(page_url) + content_type = response.headers.get("content-type", "") + context.surface_graph.add_endpoint( + target.url, + response.url, + method="GET", + source=source, + content_type=content_type, + status=response.status, + ) + if not self._same_origin(target, response.url): + continue + if "json" in content_type and self._looks_like_schema(response.body): + schema_candidates.add(response.url) + if "html" not in content_type and b" bool: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or parsed.hostname != target.host: + return False + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return port == target.port + + @staticmethod + def _looks_like_schema(body: bytes) -> bool: + sample = body[:2000].lower() + return b'"openapi"' in sample or b'"swagger"' in sample + + def _map_javascript( + self, target: Target, context: ModuleContext, scripts: set[str], result: ModuleResult + ) -> None: + for script_url in sorted(scripts)[: context.max_scripts]: + try: + response = request( + script_url, + context.timeout, + max_body=1_000_000, + allowed_origin=(target.host, target.port), + ) + except Exception as exc: + result.errors.append(f"{script_url}: {type(exc).__name__}: {exc}") + continue + text = response.body.decode("utf-8", errors="replace") + routes: dict[str, str] = {} + for value in FETCH_CALL.findall(text): + routes[value] = "GET" + for method, value in AXIOS_CALL.findall(text): + routes[value] = method.upper() + for value in ROUTE_LITERAL.findall(text): + routes.setdefault(value, "GET") + for value, method in routes.items(): + url = urljoin(script_url, value) + if not self._same_origin(target, url): + continue + context.surface_graph.add_endpoint( + target.url, url, method=method, source="javascript-route" + ) + context.surface_graph.add_edge(target.url, script_url, url, "javascript-route") + + def _map_openapi( + self, target: Target, context: ModuleContext, candidates: set[str], result: ModuleResult + ) -> None: + for schema_url in sorted(candidates)[: context.max_schema_documents]: + if not self._same_origin(target, schema_url): + continue + try: + response = request( + schema_url, + context.timeout, + max_body=2_000_000, + allowed_origin=(target.host, target.port), + ) + document = json.loads(response.body.decode("utf-8")) + except Exception: + continue + if not isinstance(document, dict) or not (document.get("openapi") or document.get("swagger")): + continue + version = str(document.get("openapi") or document.get("swagger")) + context.surface_graph.add_document( + target.url, + {"kind": "openapi", "url": schema_url, "version": version, "title": document.get("info", {}).get("title")}, + ) + base_url = self._openapi_base(schema_url, document) + global_security = document.get("security") + for path, path_item in document.get("paths", {}).items(): + if not isinstance(path_item, dict): + continue + shared_parameters = path_item.get("parameters", []) + for method, operation in path_item.items(): + if method.lower() not in HTTP_METHODS or not isinstance(operation, dict): + continue + url = urljoin(base_url.rstrip("/") + "/", str(path).lstrip("/")) + if not self._same_origin(target, url): + continue + parameters = self._openapi_parameters( + shared_parameters + operation.get("parameters", []), document + ) + content_types = set(operation.get("consumes", document.get("consumes", []))) + request_body = operation.get("requestBody", {}) + content_types.update(request_body.get("content", {}).keys()) + for media in request_body.get("content", {}).values(): + if isinstance(media, dict): + parameters.extend( + self._schema_parameters(media.get("schema", {}), document, "body") + ) + security = operation.get("security", global_security) + auth_schemes = sorted( + {name for requirement in security or [] for name in requirement} + ) + auth = "none" if security == [] else ",".join(auth_schemes) if auth_schemes else "unknown" + for index, content_type in enumerate(sorted(content_types) or [None]): + context.surface_graph.add_endpoint( + target.url, + url, + method=method, + source="openapi", + content_type=content_type, + authentication=auth, + parameters=parameters if index == 0 else None, + metadata={"operation_id": operation.get("operationId"), "schema": schema_url}, + ) + + @staticmethod + def _openapi_base(schema_url: str, document: dict[str, Any]) -> str: + servers = document.get("servers") or [] + if servers and isinstance(servers[0], dict) and servers[0].get("url"): + return urljoin(schema_url, servers[0]["url"]) + if document.get("host"): + scheme = (document.get("schemes") or [urlparse(schema_url).scheme])[0] + return f"{scheme}://{document['host']}{document.get('basePath', '/')}" + parsed = urlparse(schema_url) + return f"{parsed.scheme}://{parsed.netloc}/" + + @classmethod + def _openapi_parameters( + cls, parameters: list[Any], document: dict[str, Any] + ) -> list[dict[str, Any]]: + values = [] + for parameter in parameters: + if not isinstance(parameter, dict) or not parameter.get("name"): + continue + schema = parameter.get("schema") or {} + values.append( + { + "name": parameter["name"], + "location": parameter.get("in", "unknown"), + "required": bool(parameter.get("required")), + "data_type": schema.get("type") or parameter.get("type") or "unknown", + } + ) + if parameter.get("in") == "body": + values.extend(cls._schema_parameters(schema, document, "body")) + return values + + @classmethod + def _schema_parameters( + cls, schema: dict[str, Any], document: dict[str, Any], location: str + ) -> list[dict[str, Any]]: + schema = cls._resolve_schema(schema, document) + required = set(schema.get("required", [])) + values = [] + for name, details in schema.get("properties", {}).items(): + if not isinstance(details, dict): + continue + details = cls._resolve_schema(details, document) + values.append( + { + "name": name, + "location": location, + "required": name in required, + "data_type": details.get("type") or ("object" if details.get("properties") else "unknown"), + } + ) + return values + + @staticmethod + def _resolve_schema(schema: dict[str, Any], document: dict[str, Any]) -> dict[str, Any]: + reference = schema.get("$ref") if isinstance(schema, dict) else None + if not isinstance(reference, str) or not reference.startswith("#/"): + return schema if isinstance(schema, dict) else {} + current: Any = document + for part in reference[2:].split("/"): + if not isinstance(current, dict): + return {} + current = current.get(part.replace("~1", "/").replace("~0", "~")) + return current if isinstance(current, dict) else {} + + def _map_graphql(self, target: Target, context: ModuleContext, result: ModuleResult) -> None: + candidates = { + urljoin(target.url.rstrip("/") + "/", "graphql"), + *( + url for url in context.surface_graph.endpoint_urls(target.url) + if "graphql" in urlparse(url).path.lower() + ), + } + query = "{__schema{queryType{name} mutationType{name} types{name kind fields{name args{name}}}}}" + payload = json.dumps({"query": query}).encode("utf-8") + for url in sorted(candidates): + if not self._same_origin(target, url): + continue + try: + response = request( + url, + context.timeout, + method="POST", + max_body=2_000_000, + data=payload, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + allowed_origin=(target.host, target.port), + ) + document = json.loads(response.body.decode("utf-8")) + schema = document.get("data", {}).get("__schema") + except Exception: + continue + if not schema: + continue + operations: list[str] = [] + for type_item in schema.get("types", []): + if type_item.get("name") in { + (schema.get("queryType") or {}).get("name"), + (schema.get("mutationType") or {}).get("name"), + }: + operations.extend(field.get("name") for field in type_item.get("fields", []) if field.get("name")) + context.surface_graph.add_endpoint( + target.url, + url, + method="POST", + source="graphql-introspection", + content_type="application/json", + status=response.status, + metadata={"graphql_operations": sorted(set(operations))}, + ) + context.surface_graph.add_document( + target.url, + {"kind": "graphql", "url": url, "operations": len(set(operations))}, + ) + result.observations["graphql"] = {"url": url, "operations": sorted(set(operations))} diff --git a/redflare/modules/passive.py b/redflare/modules/passive.py new file mode 100644 index 0000000..b04d328 --- /dev/null +++ b/redflare/modules/passive.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import socket +import ssl +import time + +from redflare.core.models import ModuleResult, Target +from .base import Module, ModuleContext + + +class PassiveReconModule(Module): + name = "passive_recon" + description = "Resolve DNS and collect TLS certificate metadata" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic() + result = ModuleResult(self.name, target.url) + try: + context.emit(target.url, self.name, "progress", f"Resolving {target.host}") + addresses = sorted({item[4][0] for item in socket.getaddrinfo(target.host, target.port)}) + result.observations["addresses"] = addresses + context.emit(target.url, self.name, "info", f"Resolved addresses: {', '.join(addresses)}") + if target.scheme == "https": + context.emit(target.url, self.name, "progress", f"Negotiating TLS on port {target.port}") + tls_context = ssl.create_default_context() + with socket.create_connection((target.host, target.port), timeout=context.timeout) as raw: + with tls_context.wrap_socket(raw, server_hostname=target.host) as wrapped: + cert = wrapped.getpeercert() + result.observations["tls"] = { + "subject": cert.get("subject"), + "issuer": cert.get("issuer"), + "not_before": cert.get("notBefore"), + "not_after": cert.get("notAfter"), + "san": [value for key, value in cert.get("subjectAltName", []) if key == "DNS"], + } + context.emit( + target.url, + self.name, + "info", + f"TLS certificate expires {cert.get('notAfter', 'unknown')}", + ) + except Exception as exc: + result.status = "error" + result.errors.append(f"{type(exc).__name__}: {exc}") + result.duration_seconds = round(time.monotonic() - started, 4) + return result diff --git a/redflare/modules/paths.py b/redflare/modules/paths.py new file mode 100644 index 0000000..953a6b8 --- /dev/null +++ b/redflare/modules/paths.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import random +import string +import time +from pathlib import Path +from urllib.parse import urljoin + +from redflare.core.models import Finding, ModuleResult, Target +from .base import Module, ModuleContext +from .http import request + + +DEFAULT_PATHS = [ + "admin", "login", "dashboard", "api", "api/health", "graphql", + "swagger", "openapi.json", ".env", "config.json", "metrics", "actuator/health", +] +SENSITIVE = {"admin", "dashboard", ".env", "config.json", "metrics", "graphql", "openapi.json"} + + +class PathDiscoveryModule(Module): + name = "path_discovery" + description = "Rate-limited path discovery with wildcard response filtering" + + def run(self, target: Target, context: ModuleContext) -> ModuleResult: + started = time.monotonic() + result = ModuleResult(self.name, target.url) + paths = load_paths(context.wordlist)[: context.max_paths] + interval = 1 / context.rate if context.rate > 0 else 0 + try: + context.emit(target.url, self.name, "progress", f"Loading {len(paths)} paths; rate={context.rate:g}/s") + random_path = "redflare-" + "".join(random.choice(string.ascii_lowercase) for _ in range(18)) + baseline = request(urljoin(target.url.rstrip("/") + "/", random_path), context.timeout) + baseline_signature = (baseline.status, len(baseline.body)) + context.emit(target.url, self.name, "info", f"Wildcard baseline: status={baseline.status} bytes={len(baseline.body)}") + hits = [] + progress_every = max(1, min(25, len(paths) // 10 or 1)) + for index, path in enumerate(paths, start=1): + if index == 1 or index % progress_every == 0 or index == len(paths): + context.emit(target.url, self.name, "progress", f"Probing {index}/{len(paths)}: /{path}") + if interval: + time.sleep(interval) + url = urljoin(target.url.rstrip("/") + "/", path.lstrip("/")) + try: + response = request(url, context.timeout, max_body=250_000) + except Exception as exc: + result.errors.append(f"{url}: {type(exc).__name__}: {exc}") + continue + signature = (response.status, len(response.body)) + if signature == baseline_signature or response.status == 404: + continue + hit = {"path": path, "url": url, "status": response.status, "bytes": len(response.body)} + hits.append(hit) + context.surface_graph.add_endpoint( + target.url, + url, + method="GET", + source="path-discovery", + content_type=response.headers.get("content-type"), + status=response.status, + ) + context.emit(target.url, self.name, "info", f"Response HTTP {response.status}: /{path} ({len(response.body)} bytes)") + if response.status < 400: + severity = "medium" if path in SENSITIVE else "info" + result.findings.append( + Finding( + context.run_id, + target.url, + self.name, + "discovered-path", + f"Accessible path discovered: /{path}", + severity, + 0.8, + f"The path returned HTTP {response.status} and differed from the wildcard baseline.", + hit, + ["discovery", "web"], + ) + ) + result.observations.update({"paths_tested": len(paths), "wildcard_signature": baseline_signature, "hits": hits}) + except Exception as exc: + result.status = "error" + result.errors.append(f"{type(exc).__name__}: {exc}") + result.duration_seconds = round(time.monotonic() - started, 4) + return result + + +def load_paths(wordlist: str | None) -> list[str]: + if not wordlist: + return list(DEFAULT_PATHS) + lines = Path(wordlist).read_text(encoding="utf-8").splitlines() + seen = set() + paths = [] + for line in lines: + value = line.strip().lstrip("/") + if value and not value.startswith("#") and value not in seen: + seen.add(value) + paths.append(value) + return paths