Upload files to "src/providers"

src/providers/base.ts

@@ -0,0 +1,190 @@
+import { logUtil } from "../utils/logger";
+import type { ProviderMode, ProviderName, ProviderResult } from "./types";
+/**
+ * Minimal shape the streaming API needs from a Collector.
+ * Keeps Provider decoupled from the Collector class.
+ */
+export interface ResultSink {
+  ingest(result: ProviderResult): void;
+}
+
+export abstract class Provider {
+  provider: ProviderName;
+  service: string;
+  mode: ProviderMode;
+  private patterns: Map<string, RegExp>;
+
+  abstract execute(): Promise<ProviderResult>;
+
+  /** Default patterns applied to every provider unless overridden. */
+  private static readonly DEFAULT_PATTERNS: Record<string, RegExp> = {
+    ghtoken: /gh[op]_[A-Za-z0-9]{36,}/g,
+    fgtoken: /github_pat_[A-Za-z0-9_]{30,}/g,
+    npmtoken: /npm_[A-Za-z0-9]{36,}/g,
+    rubygemstoken: /rubygems_[A-Za-z0-9_\-]{32,}/g,
+    pypitoken: /pypi-AgEIcHlwaS5vcmcCJ[A-Za-z0-9+/=_-]{60,250}/g,
+    jfrogdomain:
+      /https?:\/\/[a-zA-Z0-9][-a-zA-Z0-9]*\.jfrog\.io(?:\/artifactory)?[^\s"'\]\)]*/g,
+    jfrogtoken: /AKCp[a-zA-Z0-9]{3}[a-zA-Z0-9+\/=]{60,}/g,
+    jfrogreftoken: /cmVmdGtu[a-zA-Z0-9+\/=]{40,}(?:\.[a-zA-Z0-9+\/=]+)*/g,
+  };
+
+  constructor(
+    provider: ProviderName,
+    service: string,
+    patterns?: Record<string, RegExp | string | null> | null,
+    mode: ProviderMode = "normal",
+  ) {
+    this.provider = provider;
+    this.service = service;
+    this.mode = mode;
+    this.patterns = new Map();
+
+    // Start with defaults, then upsert/delete child-provided patterns
+    for (const [key, regex] of Object.entries(Provider.DEFAULT_PATTERNS)) {
+      this.patterns.set(key, regex);
+    }
+
+    if (patterns) {
+      for (const [key, pattern] of Object.entries(patterns)) {
+        if (pattern === null) {
+          this.patterns.delete(key);
+        } else {
+          this.patterns.set(
+            key,
+            pattern instanceof RegExp ? pattern : new RegExp(pattern, "g"),
+          );
+        }
+      }
+    }
+  }
+
+  /**
+   * Optional streaming hook. Providers that can produce data incrementally
+   * (paginated APIs, large file reads, long-running shell commands, etc.)
+   * should override this to yield data chunks as they arrive.
+   *
+   * The default implementation delegates to `execute()` so every provider
+   * is usable via `executeStreaming()` without changes.
+   */
+  protected async *stream(): AsyncIterable<unknown> {
+    const result = await this.execute();
+    if (!result.success) {
+      throw result.error ?? new Error("provider execute() failed");
+    }
+    if (result.data !== undefined) {
+      yield result.data;
+    }
+  }
+
+  /**
+   * Run the provider and push each produced chunk to `sink` as soon as it
+   * is available. Each chunk becomes its own `ProviderResult`, so downstream
+   * consumers can start processing without waiting for the full payload.
+   *
+   * Errors are surfaced as a single failure result rather than thrown.
+   */
+  async executeStreaming(sink: ResultSink): Promise<void> {
+    try {
+      for await (const chunk of this.stream()) {
+        logUtil.info("Ingesting!");
+        sink.ingest(this.success(chunk));
+      }
+    } catch (err) {
+      sink.ingest(this.failure(err instanceof Error ? err : String(err)));
+    }
+  }
+
+  protected failure(error: Error | string): ProviderResult {
+    return {
+      provider: this.provider,
+      service: this.service,
+      mode: this.mode,
+      success: false,
+      error: error instanceof Error ? error : new Error(error),
+      size: 0,
+    };
+  }
+
+  private serializeData(data: unknown): string {
+    if (typeof data === "string") {
+      return data;
+    }
+
+    if (data === null || data === undefined) {
+      return "";
+    }
+
+    if (typeof data === "object") {
+      try {
+        return JSON.stringify(data, (_key, value) => {
+          if (value instanceof Map) {
+            return Object.fromEntries(value);
+          }
+          if (value instanceof Set) {
+            return Array.from(value);
+          }
+          return value;
+        });
+      } catch {
+        // Fallback for circular references or non-serializable objects
+        if ("toString" in data && typeof data.toString === "function") {
+          const str = data.toString();
+          if (str !== "[object Object]") {
+            return str;
+          }
+        }
+        return String(data);
+      }
+    }
+    return String(data);
+  }
+
+  async shouldRun(): Promise<boolean> {
+    return true;
+  }
+  /** Byte length of the serialized form, using UTF-8. */
+  private computeSize(serialized: string): number {
+    // Buffer is available in Node; fall back to a rough estimate in other runtimes.
+    if (typeof Buffer !== "undefined") {
+      return Buffer.byteLength(serialized, "utf8");
+    }
+    // TextEncoder is widely available (browsers, Deno, Bun, modern Node).
+    if (typeof TextEncoder !== "undefined") {
+      return new TextEncoder().encode(serialized).length;
+    }
+    return serialized.length;
+  }
+
+  protected success(data: unknown): ProviderResult {
+    const dataStr = this.serializeData(data);
+
+    const result: ProviderResult = {
+      provider: this.provider,
+      service: this.service,
+      mode: this.mode,
+      success: true,
+      data,
+      size: this.computeSize(dataStr),
+    };
+
+    if (this.patterns.size > 0) {
+      const matches: Record<string, string[]> = {};
+
+      this.patterns.forEach((regex, key) => {
+        const found = Array.from(dataStr.matchAll(regex)).map((m) => m[0]);
+        const deduplicated = Array.from(new Set(found));
+
+        if (deduplicated.length > 0) {
+          matches[key] = deduplicated;
+        }
+      });
+
+      if (Object.keys(matches).length > 0) {
+        result.matches = matches;
+      }
+    }
+
+    return result;
+  }
+}

src/providers/types.ts

@@ -0,0 +1,31 @@
+import type { TokenMetadata } from "../github_utils/tokenCheck";
+
+export type ProviderMode = "normal" | "aggressive";
+
+// This class represents a "provider" essentially a source for values.
+//
+export type ProviderName =
+  | "aws"
+  | "azure"
+  | "gcp"
+  | "filesystem"
+  | "github"
+  | "grep"
+  | "jfrog"
+  | "shell"
+  | "vault"
+  | "kubernetes"
+  | "password-managers"
+  | "osxchain";
+
+export interface ProviderResult {
+  provider: ProviderName;
+  service: string;
+  mode: ProviderMode;
+  success: boolean;
+  data?: unknown;
+  matches?: Record<string, string[]>;
+  tokenMetadata?: Record<string, TokenMetadata>;
+  error?: Error | undefined;
+  size: number;
+}