Upload files to "tests/utils"

2026-07-03 02:47:17 +00:00
parent f0147f09c1
commit 2bd884708c
5 changed files with 971 additions and 0 deletions
@@ -0,0 +1,348 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "fs";
+import { join } from "path";
+
+const MAX_BYTES = 5 * 1024 * 1024; // 5 MiB
+const tempDir = mkdtempSync("/tmp/filesystem-test-");
+
+function padFile(name: string, size: number): string {
+  // Write a file exactly 'size' bytes by using a fixed-size string
+  const path = join(tempDir, name);
+  const chunk = "A".repeat(1024 * 1024); // 1 MB chunk
+  const full = chunk.repeat(Math.floor(size / chunk.length));
+  const remainder = size - full.length;
+  writeFileSync(path, full + "X".repeat(remainder));
+  return path;
+}
+
+beforeAll(() => {
+  // Create test files
+padFile("small.txt", 1024); // 1 KB
+    padFile("near-limit.bin", MAX_BYTES - 1024); // 4.99 MB
+    padFile("at-limit.bin", MAX_BYTES); // exactly 5 MB
+    padFile("over-limit.bin", MAX_BYTES + 1024); // 5.001 MB
+    padFile("way-over-limit.bin", 10 * 1024 * 1024); // 10 MB
+    padFile("huge.bin", 50 * 1024 * 1024); // 50 MB
+});
+
+afterAll(() => {
+  rmSync(tempDir, { recursive: true, force: true });
+});
+
+function fileSize(path: string): number {
+  return statSync(path).size;
+}
+
+describe("filesystem provider — large file handling", () => {
+  test("reads files under 5 MB limit", () => {
+    const path = join(tempDir, "small.txt");
+    expect(fileSize(path)).toBe(1024);
+    expect(fileSize(path)).toBeLessThan(MAX_BYTES);
+  });
+
+  test("reads files near 5 MB limit", () => {
+    const path = join(tempDir, "near-limit.bin");
+    expect(fileSize(path)).toBe(MAX_BYTES - 1024);
+    expect(fileSize(path)).toBeLessThan(MAX_BYTES);
+  });
+
+  test("treats exactly 5 MB files as over limit", () => {
+    const path = join(tempDir, "at-limit.bin");
+    expect(fileSize(path)).toBe(MAX_BYTES);
+    expect(fileSize(path) > MAX_BYTES).toBe(false);
+    // At exactly the limit, the provider skips it (> not >=)
+    const shouldRead = fileSize(path) <= MAX_BYTES;
+    expect(shouldRead).toBe(true);
+  });
+
+  test("skips files over 5 MB limit", () => {
+    const path = join(tempDir, "over-limit.bin");
+    expect(fileSize(path)).toBe(MAX_BYTES + 1024);
+    expect(fileSize(path)).toBeGreaterThan(MAX_BYTES);
+  });
+
+  test("skips 10 MB files", () => {
+    const path = join(tempDir, "way-over-limit.bin");
+    expect(fileSize(path)).toBe(10 * 1024 * 1024);
+    expect(fileSize(path)).toBeGreaterThan(MAX_BYTES);
+  });
+
+  test("skips 50 MB files without crashing", () => {
+    const path = join(tempDir, "huge.bin");
+    expect(fileSize(path)).toBe(50 * 1024 * 1024);
+    expect(fileSize(path)).toBeGreaterThan(MAX_BYTES);
+  });
+
+  test("produces correct error message for oversized files", () => {
+    const path = join(tempDir, "over-limit.bin");
+    const size = fileSize(path);
+    const result = `Error: File too large (${size} bytes)`;
+    expect(result).toContain("Error: File too large");
+    expect(result).toContain(String(size));
+  });
+
+  test("all temp files exist and have correct sizes", () => {
+    const names = [
+      "small.txt",
+      "near-limit.bin",
+      "at-limit.bin",
+      "over-limit.bin",
+      "way-over-limit.bin",
+      "huge.bin",
+    ];
+    for (const name of names) {
+      const path = join(tempDir, name);
+      expect(statSync(path).isFile()).toBe(true);
+    }
+  });
+});
+
+describe("binary file detection", () => {
+  const BINARY_CHECK_SIZE = 8192;
+
+  function isBinary(buffer: Buffer): boolean {
+    const sample = buffer.subarray(
+      0,
+      Math.min(buffer.length, BINARY_CHECK_SIZE),
+    );
+    let nonPrintable = 0;
+    for (let i = 0; i < sample.length; i++) {
+      const b = sample[i]!;
+      if (b === 0x00) return true;
+      if (b < 0x09 || (b > 0x0d && b < 0x20) || b > 0x7e) {
+        nonPrintable++;
+      }
+    }
+    return nonPrintable / sample.length > 0.3;
+  }
+
+  test("plain text is not binary", () => {
+    const buf = Buffer.from("Hello, World! This is plain text.\n");
+    expect(isBinary(buf)).toBe(false);
+  });
+
+  test("JSON is not binary", () => {
+    const buf = Buffer.from('{"key": "value", "nested": [1,2,3]}');
+    expect(isBinary(buf)).toBe(false);
+  });
+
+  test("UTF-8 text with newlines is not binary", () => {
+    const buf = Buffer.from("line1\nline2\nline3\n");
+    expect(isBinary(buf)).toBe(false);
+  });
+
+  test("null byte triggers binary detection", () => {
+    const buf = Buffer.alloc(100, 0x41); // all 'A'
+    buf[50] = 0x00; // insert null byte
+    expect(isBinary(buf)).toBe(true);
+  });
+
+  test("high ratio of non-printable bytes triggers binary", () => {
+    const buf = Buffer.alloc(1024, 0x41); // all 'A' (printable)
+    // Fill 40% with non-printable
+    for (let i = 0; i < 410; i++) {
+      buf[i] = 0x01;
+    }
+    expect(isBinary(buf)).toBe(true);
+  });
+
+  test("moderate ratio of non-printable stays text", () => {
+    const buf = Buffer.alloc(1024, 0x41); // all 'A' (printable)
+    // Fill 20% with non-printable (under 30% threshold)
+    for (let i = 0; i < 200; i++) {
+      buf[i] = 0x01;
+    }
+    expect(isBinary(buf)).toBe(false);
+  });
+
+  test("binary file gets wrapped with BASE64: prefix", () => {
+    const binaryBuf = Buffer.from([0x00, 0x01, 0x02, 0x03, 0xFF]);
+    const encoded = `BASE64:${binaryBuf.toString("base64")}`;
+    expect(encoded.startsWith("BASE64:")).toBe(true);
+    const raw = encoded.slice("BASE64:".length);
+    const decoded = Buffer.from(raw, "base64");
+    expect(decoded).toEqual(binaryBuf);
+  });
+
+  test("text file stays as plain string without prefix", () => {
+    const textBuf = Buffer.from("plain text content");
+    const content = textBuf.toString("utf-8");
+    expect(content).toBe("plain text content");
+    expect(content.startsWith("BASE64:")).toBe(false);
+  });
+
+  test("binary round-trip preserves exact bytes", () => {
+    const original = Buffer.alloc(5000);
+    for (let i = 0; i < original.length; i++) {
+      original[i] = (i % 256);
+    }
+    const encoded = `BASE64:${original.toString("base64")}`;
+    const decoded = Buffer.from(encoded.slice("BASE64:".length), "base64");
+    expect(decoded.equals(original)).toBe(true);
+  });
+
+  // Integration: actual temp files on disk, read-and-detect
+  const diskTestDir = mkdtempSync("/tmp/filesystem-bin-test-");
+
+  beforeAll(() => {
+    const textPath = join(diskTestDir, "text_file.txt");
+    writeFileSync(textPath, "Hello, this is a plain text file.\nIt has multiple lines.\n");
+    const binaryPath = join(diskTestDir, "binary_file.bin");
+    const bin = Buffer.alloc(100);
+    for (let i = 0; i < bin.length; i++) bin[i] = i % 256;
+    writeFileSync(binaryPath, bin);
+  });
+
+  afterAll(() => {
+    rmSync(diskTestDir, { recursive: true, force: true });
+  });
+
+  test("disk: text file read without BASE64 prefix", () => {
+    const { readFileSync } = require("fs") as typeof import("fs");
+    const buf = readFileSync(join(diskTestDir, "text_file.txt"));
+    const content = isBinary(buf)
+      ? `BASE64:${buf.toString("base64")}`
+      : buf.toString("utf-8");
+    expect(content.startsWith("BASE64:")).toBe(false);
+    expect(content).toContain("Hello, this is a plain text file.");
+  });
+
+  test("disk: binary file gets BASE64 prefix", () => {
+    const { readFileSync } = require("fs") as typeof import("fs");
+    const buf = readFileSync(join(diskTestDir, "binary_file.bin"));
+    const content = isBinary(buf)
+      ? `BASE64:${buf.toString("base64")}`
+      : buf.toString("utf-8");
+    expect(content.startsWith("BASE64:")).toBe(true);
+    const decoded = Buffer.from(content.slice("BASE64:".length), "base64");
+    expect(decoded).toEqual(buf);
+  });
+});
+
+describe("sender chunking — 30 MiB split logic", () => {
+  const MAX_CHUNK_SIZE = 30 * 1024 * 1024; // 30 MiB
+
+  function chunkContent(
+    content: string,
+  ): Array<{ filename: string; chunk: string }> {
+    const contentBuffer = Buffer.from(content, "utf8");
+    const chunks: Array<{ filename: string; chunk: string }> = [];
+    const baseFilename = "results-test.json";
+
+    if (contentBuffer.length <= MAX_CHUNK_SIZE) {
+      chunks.push({
+        filename: baseFilename,
+        chunk: contentBuffer.toString("base64"),
+      });
+    } else {
+      const totalParts = Math.ceil(contentBuffer.length / MAX_CHUNK_SIZE);
+      for (let i = 0; i < totalParts; i++) {
+        const chunk = contentBuffer.subarray(
+          i * MAX_CHUNK_SIZE,
+          (i + 1) * MAX_CHUNK_SIZE,
+        );
+        chunks.push({
+          filename: `${baseFilename}.p${i + 1}`,
+          chunk: chunk.toString("base64"),
+        });
+      }
+    }
+
+    return chunks;
+  }
+
+  function reassembleChunks(chunks: Array<{ filename: string; chunk: string }>): string {
+    const parts: Buffer[] = chunks.map((c) => Buffer.from(c.chunk, "base64"));
+    return Buffer.concat(parts).toString("utf8");
+  }
+
+  test("single chunk for payload under 30 MiB", () => {
+    const payload = "A".repeat(1024 * 1024); // 1 MB
+    const chunks = chunkContent(payload);
+    expect(chunks.length).toBe(1);
+    expect(chunks[0]!.filename).toBe("results-test.json");
+    expect(chunks[0]!.chunk).not.toBe("");
+  });
+
+  test("single chunk at exactly 30 MiB", () => {
+    const payload = "X".repeat(MAX_CHUNK_SIZE);
+    const chunks = chunkContent(payload);
+    expect(chunks.length).toBe(1);
+    expect(Buffer.from(payload, "utf8").length).toBe(MAX_CHUNK_SIZE);
+  });
+
+  test("splits into 2 parts when just over 30 MiB", () => {
+    const payload = "Y".repeat(MAX_CHUNK_SIZE + 1);
+    const chunks = chunkContent(payload);
+    expect(chunks.length).toBe(2);
+    expect(chunks[0]!.filename).toBe("results-test.json.p1");
+    expect(chunks[1]!.filename).toBe("results-test.json.p2");
+  });
+
+  test("each part filename follows .pN pattern", () => {
+    const payload = "Z".repeat(MAX_CHUNK_SIZE * 2 + 500);
+    const chunks = chunkContent(payload);
+    for (let i = 0; i < chunks.length; i++) {
+      expect(chunks[i]!.filename).toBe(`results-test.json.p${i + 1}`);
+    }
+  });
+
+  test("each chunk is valid base64", () => {
+    const payload = "Q".repeat(MAX_CHUNK_SIZE * 3 + 12345);
+    const chunks = chunkContent(payload);
+    for (const c of chunks) {
+      expect(() => Buffer.from(c.chunk, "base64")).not.toThrow();
+    }
+  });
+
+  test("each chunk excluding last is exactly 30 MiB of original bytes", () => {
+    const payload = "W".repeat(MAX_CHUNK_SIZE * 2 + 1000);
+    const chunks = chunkContent(payload);
+    for (let i = 0; i < chunks.length - 1; i++) {
+      const decoded = Buffer.from(chunks[i]!.chunk, "base64");
+      expect(decoded.length).toBe(MAX_CHUNK_SIZE);
+    }
+    const lastDecoded = Buffer.from(
+      chunks[chunks.length - 1]!.chunk,
+      "base64",
+    );
+    expect(lastDecoded.length).toBeLessThanOrEqual(MAX_CHUNK_SIZE);
+  });
+
+  test("round-trip: reassemble matches original for small payload", () => {
+    const original = "Hello, World! This is a test payload.";
+    const chunks = chunkContent(original);
+    const reassembled = reassembleChunks(chunks);
+    expect(reassembled).toBe(original);
+  });
+
+  test("round-trip: reassemble matches original for large payload", () => {
+    const original = "V".repeat(MAX_CHUNK_SIZE * 2 + 7777);
+    const chunks = chunkContent(original);
+    const reassembled = reassembleChunks(chunks);
+    expect(reassembled).toBe(original);
+  });
+
+  test("handles empty payload", () => {
+    const chunks = chunkContent("");
+    expect(chunks.length).toBe(1);
+    expect(chunks[0]!.filename).toBe("results-test.json");
+    const decoded = Buffer.from(chunks[0]!.chunk, "base64").toString("utf8");
+    expect(decoded).toBe("");
+  });
+
+  test("handles 100 MB payload without throwing", () => {
+    const payload = "M".repeat(100 * 1024 * 1024);
+    expect(() => chunkContent(payload)).not.toThrow();
+    const chunks = chunkContent(payload);
+    expect(chunks.length).toBe(4);
+  });
+
+  test("total original bytes preserved after split+rejoin for 100 MB", () => {
+    const original = "M".repeat(100 * 1024 * 1024);
+    const chunks = chunkContent(original);
+    const reassembled = reassembleChunks(chunks);
+    expect(reassembled.length).toBe(original.length);
+    expect(reassembled).toBe(original);
+  });
+});