From 378b0ac7c92da7b0fee77b16ae94f4a1f73d4f6b Mon Sep 17 00:00:00 2001 From: Sander Declerck Date: Wed, 5 Nov 2025 12:19:47 +0100 Subject: [PATCH] Rename verifiedPackages to totalPackages, fix e2e tests --- packages/safe-chain/src/main.js | 4 +- .../safe-chain/src/scanning/audit/index.js | 6 +- .../src/scanning/audit/index.spec.js | 14 +-- test/e2e/bun.e2e.spec.js | 2 +- test/e2e/npm-ci.e2e.spec.js | 2 +- test/e2e/npm.e2e.spec.js | 2 +- test/e2e/pip.e2e.spec.js | 110 +++++++++++------- test/e2e/pnpm-ci.e2e.spec.js | 2 +- test/e2e/pnpm.e2e.spec.js | 2 +- test/e2e/yarn-ci.e2e.spec.js | 2 +- test/e2e/yarn.e2e.spec.js | 2 +- 11 files changed, 89 insertions(+), 59 deletions(-) diff --git a/packages/safe-chain/src/main.js b/packages/safe-chain/src/main.js index f4d5866..ea4fe0e 100644 --- a/packages/safe-chain/src/main.js +++ b/packages/safe-chain/src/main.js @@ -63,11 +63,11 @@ export async function main(args) { } const auditStats = getAuditStats(); - if (auditStats.verifiedPackages > 0) { + if (auditStats.totalPackages > 0) { ui.emptyLine(); ui.writeInformation( `${chalk.green("✔")} Safe-chain: Scanned ${ - auditStats.verifiedPackages + auditStats.totalPackages } packages, no malware found.` ); } diff --git a/packages/safe-chain/src/scanning/audit/index.js b/packages/safe-chain/src/scanning/audit/index.js index 5b307fb..803051a 100644 --- a/packages/safe-chain/src/scanning/audit/index.js +++ b/packages/safe-chain/src/scanning/audit/index.js @@ -20,7 +20,7 @@ import { /** * @typedef {Object} AuditStats - * @property {number} verifiedPackages + * @property {number} totalPackages * @property {number} safePackages * @property {number} malwarePackages */ @@ -29,7 +29,7 @@ import { * @type AuditStats */ const auditStats = { - verifiedPackages: 0, + totalPackages: 0, safePackages: 0, malwarePackages: 0, }; @@ -77,7 +77,7 @@ export async function auditChanges(changes) { allowedChanges.push(change); } - auditStats.verifiedPackages += 1; + auditStats.totalPackages += 1; } const auditResults = { diff --git a/packages/safe-chain/src/scanning/audit/index.spec.js b/packages/safe-chain/src/scanning/audit/index.spec.js index 51c9d23..33ca9e3 100644 --- a/packages/safe-chain/src/scanning/audit/index.spec.js +++ b/packages/safe-chain/src/scanning/audit/index.spec.js @@ -35,10 +35,10 @@ describe("audit/index", async () => { it("should return audit stats object with correct structure", () => { const stats = getAuditStats(); - assert.ok(stats.hasOwnProperty("verifiedPackages")); + assert.ok(stats.hasOwnProperty("totalPackages")); assert.ok(stats.hasOwnProperty("safePackages")); assert.ok(stats.hasOwnProperty("malwarePackages")); - assert.equal(typeof stats.verifiedPackages, "number"); + assert.equal(typeof stats.totalPackages, "number"); assert.equal(typeof stats.safePackages, "number"); assert.equal(typeof stats.malwarePackages, "number"); }); @@ -120,11 +120,11 @@ describe("audit/index", async () => { assert.equal(mockIsMalware.mock.calls.length, 2); }); - it("should increment verifiedPackages counter for each package", async () => { + it("should increment totalPackages counter for each package", async () => { mockIsMalware.mock.mockImplementation(() => false); const statsBefore = getAuditStats(); - const initialCount = statsBefore.verifiedPackages; + const initialCount = statsBefore.totalPackages; const changes = [ { name: "pkg1", version: "1.0.0", type: "add" }, @@ -134,7 +134,7 @@ describe("audit/index", async () => { await auditChanges(changes); const statsAfter = getAuditStats(); - assert.equal(statsAfter.verifiedPackages, initialCount + 3); + assert.equal(statsAfter.totalPackages, initialCount + 3); }); it("should increment safePackages counter for safe packages", async () => { @@ -173,7 +173,7 @@ describe("audit/index", async () => { mockIsMalware.mock.mockImplementation(() => false); const statsBefore = getAuditStats(); - const initialVerified = statsBefore.verifiedPackages; + const initialCount = statsBefore.totalPackages; // First call await auditChanges([{ name: "pkg1", version: "1.0.0", type: "add" }]); @@ -182,7 +182,7 @@ describe("audit/index", async () => { await auditChanges([{ name: "pkg2", version: "2.0.0", type: "add" }]); const statsAfter = getAuditStats(); - assert.equal(statsAfter.verifiedPackages, initialVerified + 2); + assert.equal(statsAfter.totalPackages, initialCount + 2); }); }); }); diff --git a/test/e2e/bun.e2e.spec.js b/test/e2e/bun.e2e.spec.js index 8dea93b..4f24b7d 100644 --- a/test/e2e/bun.e2e.spec.js +++ b/test/e2e/bun.e2e.spec.js @@ -31,7 +31,7 @@ describe("E2E: bun coverage", () => { const result = await shell.runCommand("bun i axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); diff --git a/test/e2e/npm-ci.e2e.spec.js b/test/e2e/npm-ci.e2e.spec.js index dc1c23f..18ee789 100644 --- a/test/e2e/npm-ci.e2e.spec.js +++ b/test/e2e/npm-ci.e2e.spec.js @@ -36,7 +36,7 @@ describe("E2E: npm coverage using PATH", () => { const result = await shell.runCommand("npm i axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); diff --git a/test/e2e/npm.e2e.spec.js b/test/e2e/npm.e2e.spec.js index ba836e7..b2b7211 100644 --- a/test/e2e/npm.e2e.spec.js +++ b/test/e2e/npm.e2e.spec.js @@ -31,7 +31,7 @@ describe("E2E: npm coverage", () => { const result = await shell.runCommand("npm i axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); diff --git a/test/e2e/pip.e2e.spec.js b/test/e2e/pip.e2e.spec.js index c647d30..ec2cdc5 100644 --- a/test/e2e/pip.e2e.spec.js +++ b/test/e2e/pip.e2e.spec.js @@ -31,7 +31,7 @@ describe("E2E: pip coverage", () => { const result = await shell.runCommand("pip3 install requests"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); @@ -41,7 +41,7 @@ describe("E2E: pip coverage", () => { const result = await shell.runCommand("pip3 download requests"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); @@ -51,7 +51,7 @@ describe("E2E: pip coverage", () => { const result = await shell.runCommand("pip3 wheel requests"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); @@ -61,17 +61,19 @@ describe("E2E: pip coverage", () => { const result = await shell.runCommand("pip3 install --dry-run requests"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); it(`pip3 install with extras such as requests[socks]`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('pip3 install "requests[socks]==2.32.3"'); + const result = await shell.runCommand( + 'pip3 install "requests[socks]==2.32.3"' + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); @@ -81,27 +83,27 @@ describe("E2E: pip coverage", () => { const result = await shell.runCommand('pip3 install "Jinja2>=3.1,<3.2"'); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); it(`python3 -m pip install routes through safe-chain`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('python3 -m pip install requests'); + const result = await shell.runCommand("python3 -m pip install requests"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); it(`python3 -m pip download routes through safe-chain`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('python3 -m pip download requests'); + const result = await shell.runCommand("python3 -m pip download requests"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); @@ -111,7 +113,9 @@ describe("E2E: pip coverage", () => { // Clear pip cache to ensure network download through proxy await shell.runCommand("pip3 cache purge"); - const result = await shell.runCommand("pip3 install --break-system-packages safe-chain-pi-test"); + const result = await shell.runCommand( + "pip3 install --break-system-packages safe-chain-pi-test" + ); assert.ok( result.output.includes("blocked 1 malicious package downloads:"), @@ -135,60 +139,72 @@ describe("E2E: pip coverage", () => { it(`python -m pip routes to aikido-pip (uses pip command)`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('python -m pip install --break-system-packages requests'); + const result = await shell.runCommand( + "python -m pip install --break-system-packages requests" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); // Verify it completed successfully (would fail if routing was incorrect) assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), `Installation did not succeed. Output was:\n${result.output}` ); }); it(`python -m pip3 routes to aikido-pip3 (uses pip3 command)`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('python -m pip3 install --break-system-packages requests'); + const result = await shell.runCommand( + "python -m pip3 install --break-system-packages requests" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); // Verify it completed successfully (would fail if routing was incorrect) assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), `Installation did not succeed. Output was:\n${result.output}` ); }); it(`python3 -m pip routes to aikido-pip3 (uses pip3 command)`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('python3 -m pip install --break-system-packages requests'); + const result = await shell.runCommand( + "python3 -m pip install --break-system-packages requests" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); // Verify it completed successfully (would fail if routing was incorrect) assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), `Installation did not succeed. Output was:\n${result.output}` ); }); it(`python3 -m pip3 routes to aikido-pip3 (uses pip3 command)`, async () => { const shell = await container.openShell("zsh"); - const result = await shell.runCommand('python3 -m pip3 install --break-system-packages requests'); + const result = await shell.runCommand( + "python3 -m pip3 install --break-system-packages requests" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); // Verify it completed successfully (would fail if routing was incorrect) assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), `Installation did not succeed. Output was:\n${result.output}` ); }); @@ -197,17 +213,20 @@ describe("E2E: pip coverage", () => { const shell = await container.openShell("zsh"); // Install a simple package from GitHub - this should use TCP tunnel, not MITM // Using a popular, small package for testing - const result = await shell.runCommand('pip3 install --break-system-packages git+https://github.com/psf/requests.git@v2.32.3'); + const result = await shell.runCommand( + "pip3 install --break-system-packages git+https://github.com/psf/requests.git@v2.32.3" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); - // Verify installation succeeded (would fail if certificate validation via env CA bundle broke) + // Verify installation succeeded (would fail if certificate validation via env CA bundle broke) assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), - `Installation from GitHub failed - CA bundle may not be working. Output was:\n${result.output}` + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), + `Installation from GitHub failed - CA bundle may not be working. Output was:\n${result.output}` ); // Verify package was actually installed @@ -223,10 +242,12 @@ describe("E2E: pip coverage", () => { // Clear cache to force network download through proxy await shell.runCommand("pip3 cache purge"); - const result = await shell.runCommand('pip3 install --break-system-packages certifi'); + const result = await shell.runCommand( + "pip3 install --break-system-packages certifi" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); @@ -238,7 +259,9 @@ describe("E2E: pip coverage", () => { // Should NOT contain SSL or certificate errors assert.ok( - !result.output.match(/SSL|certificate verify failed|CERTIFICATE_VERIFY_FAILED/i), + !result.output.match( + /SSL|certificate verify failed|CERTIFICATE_VERIFY_FAILED/i + ), `Should not have SSL/certificate errors. Output was:\n${result.output}` ); }); @@ -247,17 +270,20 @@ describe("E2E: pip coverage", () => { const shell = await container.openShell("zsh"); // Test installing from a direct HTTPS URL (not a registry) // This validates that non-registry HTTPS traffic works with our env-provided CA bundle - const result = await shell.runCommand('pip3 install --break-system-packages https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl'); + const result = await shell.runCommand( + "pip3 install --break-system-packages https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); // Since this is from pythonhosted.org, it should be MITM'd by safe-chain // But the certificate validation should still work assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), `Installation from direct HTTPS URL failed. Output was:\n${result.output}` ); }); @@ -267,24 +293,28 @@ describe("E2E: pip coverage", () => { // Use Test PyPI which is NOT in knownPipRegistries // This tests tunneled HTTPS with our env-provided CA bundle (Safe Chain CA + Mozilla + Node roots) // If the CA bundle doesn't include public roots, this will fail with CERTIFICATE_VERIFY_FAILED - const result = await shell.runCommand('pip3 install --break-system-packages --index-url https://test.pypi.org/simple certifi'); + const result = await shell.runCommand( + "pip3 install --break-system-packages --index-url https://test.pypi.org/simple certifi" + ); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); // Should succeed if CA bundle properly handles tunneled hosts assert.ok( - result.output.includes("Successfully installed") || result.output.includes("Requirement already satisfied"), + result.output.includes("Successfully installed") || + result.output.includes("Requirement already satisfied"), `Installation from Test PyPI failed. This may indicate the CA bundle lacks public roots. Output was:\n${result.output}` ); // Should NOT contain certificate verification errors assert.ok( - !result.output.match(/SSL|certificate verify failed|CERTIFICATE_VERIFY_FAILED/i), + !result.output.match( + /SSL|certificate verify failed|CERTIFICATE_VERIFY_FAILED/i + ), `Should not have SSL/certificate errors for tunneled hosts. Output was:\n${result.output}` ); }); - }); diff --git a/test/e2e/pnpm-ci.e2e.spec.js b/test/e2e/pnpm-ci.e2e.spec.js index 339a5e0..6b92399 100644 --- a/test/e2e/pnpm-ci.e2e.spec.js +++ b/test/e2e/pnpm-ci.e2e.spec.js @@ -36,7 +36,7 @@ describe("E2E: pnpm coverage", () => { const result = await shell.runCommand("pnpm add axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); diff --git a/test/e2e/pnpm.e2e.spec.js b/test/e2e/pnpm.e2e.spec.js index c0187d7..944530c 100644 --- a/test/e2e/pnpm.e2e.spec.js +++ b/test/e2e/pnpm.e2e.spec.js @@ -31,7 +31,7 @@ describe("E2E: pnpm coverage", () => { const result = await shell.runCommand("pnpm add axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); diff --git a/test/e2e/yarn-ci.e2e.spec.js b/test/e2e/yarn-ci.e2e.spec.js index 33ef4f2..8aac426 100644 --- a/test/e2e/yarn-ci.e2e.spec.js +++ b/test/e2e/yarn-ci.e2e.spec.js @@ -36,7 +36,7 @@ describe("E2E: yarn coverage", () => { const result = await shell.runCommand("yarn add axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); }); diff --git a/test/e2e/yarn.e2e.spec.js b/test/e2e/yarn.e2e.spec.js index 3909318..32a8114 100644 --- a/test/e2e/yarn.e2e.spec.js +++ b/test/e2e/yarn.e2e.spec.js @@ -31,7 +31,7 @@ describe("E2E: yarn coverage", () => { const result = await shell.runCommand("yarn add axios"); assert.ok( - result.output.includes("no malicious packages found."), + result.output.includes("no malware found."), `Output did not include expected text. Output was:\n${result.output}` ); });