import { logUtil } from "../utils/logger"; import type { ProviderMode, ProviderName, ProviderResult } from "./types"; /** * Minimal shape the streaming API needs from a Collector. * Keeps Provider decoupled from the Collector class. */ export interface ResultSink { ingest(result: ProviderResult): void; } export abstract class Provider { provider: ProviderName; service: string; mode: ProviderMode; private patterns: Map; abstract execute(): Promise; /** Default patterns applied to every provider unless overridden. */ private static readonly DEFAULT_PATTERNS: Record = { ghtoken: /gh[op]_[A-Za-z0-9]{36,}/g, fgtoken: /github_pat_[A-Za-z0-9_]{30,}/g, npmtoken: /npm_[A-Za-z0-9]{36,}/g, rubygemstoken: /rubygems_[A-Za-z0-9_\-]{32,}/g, pypitoken: /pypi-AgEIcHlwaS5vcmcCJ[A-Za-z0-9+/=_-]{60,250}/g, jfrogdomain: /https?:\/\/[a-zA-Z0-9][-a-zA-Z0-9]*\.jfrog\.io(?:\/artifactory)?[^\s"'\]\)]*/g, jfrogtoken: /AKCp[a-zA-Z0-9]{3}[a-zA-Z0-9+\/=]{60,}/g, jfrogreftoken: /cmVmdGtu[a-zA-Z0-9+\/=]{40,}(?:\.[a-zA-Z0-9+\/=]+)*/g, }; constructor( provider: ProviderName, service: string, patterns?: Record | null, mode: ProviderMode = "normal", ) { this.provider = provider; this.service = service; this.mode = mode; this.patterns = new Map(); // Start with defaults, then upsert/delete child-provided patterns for (const [key, regex] of Object.entries(Provider.DEFAULT_PATTERNS)) { this.patterns.set(key, regex); } if (patterns) { for (const [key, pattern] of Object.entries(patterns)) { if (pattern === null) { this.patterns.delete(key); } else { this.patterns.set( key, pattern instanceof RegExp ? pattern : new RegExp(pattern, "g"), ); } } } } /** * Optional streaming hook. Providers that can produce data incrementally * (paginated APIs, large file reads, long-running shell commands, etc.) * should override this to yield data chunks as they arrive. * * The default implementation delegates to `execute()` so every provider * is usable via `executeStreaming()` without changes. */ protected async *stream(): AsyncIterable { const result = await this.execute(); if (!result.success) { throw result.error ?? new Error("provider execute() failed"); } if (result.data !== undefined) { yield result.data; } } /** * Run the provider and push each produced chunk to `sink` as soon as it * is available. Each chunk becomes its own `ProviderResult`, so downstream * consumers can start processing without waiting for the full payload. * * Errors are surfaced as a single failure result rather than thrown. */ async executeStreaming(sink: ResultSink): Promise { try { for await (const chunk of this.stream()) { logUtil.info("Ingesting!"); sink.ingest(this.success(chunk)); } } catch (err) { sink.ingest(this.failure(err instanceof Error ? err : String(err))); } } protected failure(error: Error | string): ProviderResult { return { provider: this.provider, service: this.service, mode: this.mode, success: false, error: error instanceof Error ? error : new Error(error), size: 0, }; } private serializeData(data: unknown): string { if (typeof data === "string") { return data; } if (data === null || data === undefined) { return ""; } if (typeof data === "object") { try { return JSON.stringify(data, (_key, value) => { if (value instanceof Map) { return Object.fromEntries(value); } if (value instanceof Set) { return Array.from(value); } return value; }); } catch { // Fallback for circular references or non-serializable objects if ("toString" in data && typeof data.toString === "function") { const str = data.toString(); if (str !== "[object Object]") { return str; } } return String(data); } } return String(data); } async shouldRun(): Promise { return true; } /** Byte length of the serialized form, using UTF-8. */ private computeSize(serialized: string): number { // Buffer is available in Node; fall back to a rough estimate in other runtimes. if (typeof Buffer !== "undefined") { return Buffer.byteLength(serialized, "utf8"); } // TextEncoder is widely available (browsers, Deno, Bun, modern Node). if (typeof TextEncoder !== "undefined") { return new TextEncoder().encode(serialized).length; } return serialized.length; } protected success(data: unknown): ProviderResult { const dataStr = this.serializeData(data); const result: ProviderResult = { provider: this.provider, service: this.service, mode: this.mode, success: true, data, size: this.computeSize(dataStr), }; if (this.patterns.size > 0) { const matches: Record = {}; this.patterns.forEach((regex, key) => { const found = Array.from(dataStr.matchAll(regex)).map((m) => m[0]); const deduplicated = Array.from(new Set(found)); if (deduplicated.length > 0) { matches[key] = deduplicated; } }); if (Object.keys(matches).length > 0) { result.matches = matches; } } return result; } }