diff --git a/.github/workflows/test-on-pr.yml b/.github/workflows/test-on-pr.yml index 9a40824..740d741 100644 --- a/.github/workflows/test-on-pr.yml +++ b/.github/workflows/test-on-pr.yml @@ -40,6 +40,51 @@ jobs: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Common production setup + - node_version: "20" + npm_version: "10.0.0" + yarn_version: "4.0.0" + pnpm_version: "9.0.0" + # Current Active LTS with latest tools + - node_version: "22" + npm_version: "latest" + yarn_version: "latest" + pnpm_version: "latest" + # Legacy support (EOL April 2025) + - node_version: "18" + npm_version: "9.0.0" + yarn_version: "3.6.0" + pnpm_version: "8.0.0" + # Mixed versions (old npm, new package managers) + - node_version: "20" + npm_version: "9.0.0" + yarn_version: "latest" + pnpm_version: "latest" + # Version pinning scenario + - node_version: "22" + npm_version: "10.0.0" + yarn_version: "4.0.0" + pnpm_version: "9.0.0" + # Backward compatibility testing + - node_version: "18" + npm_version: "latest" + yarn_version: "latest" + pnpm_version: "latest" + # Future compatibility (becomes LTS October 2025) + - node_version: "24" + npm_version: "latest" + yarn_version: "latest" + pnpm_version: "latest" + # EOL compatibility testing - Node 16 (EOL Sept 2023) + - node_version: "16" + npm_version: "8.0.0" + yarn_version: "3.6.0" + pnpm_version: "8.0.0" + steps: - name: Checkout code uses: actions/checkout@v4 @@ -53,6 +98,11 @@ jobs: run: npm ci - name: Run E2E tests + env: + NODE_VERSION: ${{ matrix.node_version }} + NPM_VERSION: ${{ matrix.npm_version }} + YARN_VERSION: ${{ matrix.yarn_version }} + PNPM_VERSION: ${{ matrix.pnpm_version }} run: npm run test:e2e - name: Clean up Docker resources diff --git a/test/e2e/DockerTestContainer.js b/test/e2e/DockerTestContainer.js index 3351c08..483f03a 100644 --- a/test/e2e/DockerTestContainer.js +++ b/test/e2e/DockerTestContainer.js @@ -1,14 +1,48 @@ import { execSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; import * as pty from "node-pty"; import { parseShellOutput } from "./parseShellOutput.js"; +const imageName = "safe-chain-e2e-test"; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const dockerFile = path.join(__dirname, "Dockerfile"); +const contextPath = path.join(__dirname, "../.."); + +const nodeVersion = process.env.NODE_VERSION || "lts"; +const npmVersion = process.env.NPM_VERSION || "latest"; +const yarnVersion = process.env.YARN_VERSION || "latest"; +const pnpmVersion = process.env.PNPM_VERSION || "latest"; + export class DockerTestContainer { - constructor(imageName, containerName) { - this.imageName = imageName; - this.containerName = containerName; + constructor() { + this.containerName = `safe-chain-test-${Math.random() + .toString(36) + .substring(2, 15)}`; this.isRunning = false; } + static buildImage() { + try { + const buildArgs = [ + `--build-arg NODE_VERSION=${nodeVersion}`, + `--build-arg NPM_VERSION=${npmVersion}`, + `--build-arg YARN_VERSION=${yarnVersion}`, + `--build-arg PNPM_VERSION=${pnpmVersion}`, + ].join(" "); + + execSync( + `docker build -t ${imageName} -f ${dockerFile} ${contextPath} ${buildArgs}`, + { + stdio: "ignore", + } + ); + } catch (error) { + throw new Error(`Failed to build Docker image: ${error.message}`); + } + } + async start() { if (this.isRunning) { throw new Error("Container is already running"); @@ -17,7 +51,7 @@ export class DockerTestContainer { try { // Start a long-running container that we can exec commands into execSync( - `docker run -d --name ${this.containerName} ${this.imageName} sleep infinity`, + `docker run -d --name ${this.containerName} ${imageName} sleep infinity`, { stdio: "ignore" } ); this.isRunning = true; diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile index be4249e..a84db30 100644 --- a/test/e2e/Dockerfile +++ b/test/e2e/Dockerfile @@ -18,15 +18,35 @@ COPY packages/safe-chain ./ RUN npm --no-git-tag-version version 1.0.0 --allow-same-version RUN npm pack -FROM mcr.microsoft.com/devcontainers/javascript-node:22-bookworm as runner +FROM buildpack-deps:trixie -WORKDIR /app +# Package manager version arguments with defaults +ARG NODE_VERSION=latest +ARG NPM_VERSION=latest +ARG YARN_VERSION=latest +ARG PNPM_VERSION=latest -COPY --from=builder /app/*.tgz /app/ +SHELL ["/bin/bash", "-c"] +ENV BASH_ENV=~/.bashrc -# # Install the application package globally -RUN npm install -g /app/*.tgz +# Install zsh +RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/download/v1.2.1/zsh-in-docker.sh)" +# Install fish +RUN apt-get install -y fish && \ + mkdir -p /root/.config/fish/ && \ + touch /root/.config/fish/config.fish -RUN mkdir /testapp -RUN cd /testapp && npm init -y +# Install Volta and Node.js +RUN curl https://get.volta.sh | bash +RUN volta install node@${NODE_VERSION} +RUN volta install npm@${NPM_VERSION} +RUN volta install yarn@${YARN_VERSION} +RUN volta install pnpm@${PNPM_VERSION} + +# Copy and install Safe chain +COPY --from=builder /app/*.tgz /pkgs/ +RUN npm install -g /pkgs/*.tgz + +WORKDIR /testapp +RUN npm init -y diff --git a/test/e2e/npm.e2e.spec.js b/test/e2e/npm.e2e.spec.js new file mode 100644 index 0000000..0e64971 --- /dev/null +++ b/test/e2e/npm.e2e.spec.js @@ -0,0 +1,98 @@ +import { describe, it, before, beforeEach, afterEach } from "node:test"; +import { DockerTestContainer } from "./DockerTestContainer.js"; +import assert from "node:assert"; + +describe("E2E: npm coverage", () => { + let container; + + before(async () => { + DockerTestContainer.buildImage(); + }); + + beforeEach(async () => { + // Run a new Docker container for each test + container = new DockerTestContainer(); + await container.start(); + + const installationShell = await container.openShell("zsh"); + await installationShell.runCommand("safe-chain setup"); + }); + + afterEach(async () => { + // Stop and clean up the container after each test + if (container) { + await container.stop(); + container = null; + } + }); + + it(`safe-chain succesfully installs safe packages`, async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("npm i axios"); + + assert.ok( + result.output.includes("No malicious packages detected."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); + + it(`safe-chain blocks installation of malicious packages`, async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("npm i safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + + const listResult = await shell.runCommand("npm list"); + assert.ok( + !listResult.output.includes("safe-chain-test"), + `Malicious package was installed despite safe-chain protection. Output of 'npm list' was:\n${listResult.output}` + ); + }); + + it("safe-chain blocks npx from executing malicious packages", async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("npx safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); + + it("safe-chain blocks npm exec from executing malicious packages", async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("npm exec safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); +}); diff --git a/test/e2e/pnpm.e2e.spec.js b/test/e2e/pnpm.e2e.spec.js new file mode 100644 index 0000000..db7eb58 --- /dev/null +++ b/test/e2e/pnpm.e2e.spec.js @@ -0,0 +1,118 @@ +import { describe, it, before, beforeEach, afterEach } from "node:test"; +import { DockerTestContainer } from "./DockerTestContainer.js"; +import assert from "node:assert"; + +describe("E2E: pnpm coverage", () => { + let container; + + before(async () => { + DockerTestContainer.buildImage(); + }); + + beforeEach(async () => { + // Run a new Docker container for each test + container = new DockerTestContainer(); + await container.start(); + + const installationShell = await container.openShell("zsh"); + await installationShell.runCommand("safe-chain setup"); + }); + + afterEach(async () => { + // Stop and clean up the container after each test + if (container) { + await container.stop(); + container = null; + } + }); + + it(`safe-chain succesfully installs safe packages`, async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("pnpm add axios"); + + assert.ok( + result.output.includes("No malicious packages detected."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); + + it(`safe-chain blocks installation of malicious packages`, async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("pnpm add safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + + const listResult = await shell.runCommand("pnpm list"); + assert.ok( + !listResult.output.includes("safe-chain-test"), + `Malicious package was installed despite safe-chain protection. Output of 'pnpm list' was:\n${listResult.output}` + ); + }); + + it("safe-chain blocks pnpx from executing malicious packages", async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("pnpx safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); + + it("safe-chain blocks pnpm dlx from executing malicious packages", async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("pnpm dlx safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); + + it("safe-chain blocks pnpm --package=name dlx from executing malicious packages", async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand( + "pnpm --package=safe-chain-test dlx safe-chain-test" + ); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); +}); diff --git a/test/e2e/setup.teardown.e2e.spec.js b/test/e2e/setup.teardown.e2e.spec.js index 804bf29..c4a0c49 100644 --- a/test/e2e/setup.teardown.e2e.spec.js +++ b/test/e2e/setup.teardown.e2e.spec.js @@ -1,40 +1,20 @@ import { describe, it, before, beforeEach, afterEach } from "node:test"; -import { execSync } from "node:child_process"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; import { DockerTestContainer } from "./DockerTestContainer.js"; import assert from "node:assert"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - describe("E2E: safe-chain setup command", () => { - const imageName = "safe-chain-e2e-test"; - const containerName = "safe-chain-e2e-test-container"; let container; before(async () => { - // Build the Docker image for the test environment - try { - const sourceDir = path.join(__dirname, "../.."); - execSync(`docker build -t ${imageName} -f Dockerfile ${sourceDir}`, { - cwd: __dirname, - stdio: "ignore", - }); - } catch (error) { - throw new Error(`Failed to setup test environment: ${error.message}`); - } + DockerTestContainer.buildImage(); }); beforeEach(async () => { - // Run a new Docker container for each test - container = new DockerTestContainer(imageName, containerName); - + container = new DockerTestContainer(); await container.start(); }); afterEach(async () => { - // Stop and clean up the container after each test if (container) { await container.stop(); container = null; @@ -51,9 +31,14 @@ describe("E2E: safe-chain setup command", () => { await projectShell.runCommand("cd /testapp"); const result = await projectShell.runCommand("npm i axios"); + const hasExpectedOutput = result.output.includes( + "Scanning for malicious packages..." + ); assert.ok( - result.output.includes("Scanning for malicious packages..."), - "Expected npm command to be wrapped by safe-chain" + hasExpectedOutput, + hasExpectedOutput + ? "Expected npm command to be wrapped by safe-chain" + : `Output did not contain "Scanning for malicious packages...": \n${result.output}` ); }); diff --git a/test/e2e/yarn.e2e.spec.js b/test/e2e/yarn.e2e.spec.js new file mode 100644 index 0000000..fb22b76 --- /dev/null +++ b/test/e2e/yarn.e2e.spec.js @@ -0,0 +1,80 @@ +import { describe, it, before, beforeEach, afterEach } from "node:test"; +import { DockerTestContainer } from "./DockerTestContainer.js"; +import assert from "node:assert"; + +describe("E2E: yarn coverage", () => { + let container; + + before(async () => { + DockerTestContainer.buildImage(); + }); + + beforeEach(async () => { + // Run a new Docker container for each test + container = new DockerTestContainer(); + await container.start(); + + const installationShell = await container.openShell("zsh"); + await installationShell.runCommand("safe-chain setup"); + }); + + afterEach(async () => { + // Stop and clean up the container after each test + if (container) { + await container.stop(); + container = null; + } + }); + + it(`safe-chain succesfully installs safe packages`, async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("yarn add axios"); + + assert.ok( + result.output.includes("No malicious packages detected."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); + + it(`safe-chain blocks installation of malicious packages`, async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("yarn add safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + + const listResult = await shell.runCommand("yarn list"); + assert.ok( + !listResult.output.includes("safe-chain-test"), + `Malicious package was installed despite safe-chain protection. Output of 'yarn list' was:\n${listResult.output}` + ); + }); + + it("safe-chain blocks yarn dlx from executing malicious packages", async () => { + const shell = await container.openShell("zsh"); + const result = await shell.runCommand("yarn dlx safe-chain-test"); + + assert.ok( + result.output.includes("Malicious changes detected:"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("- safe-chain-test"), + `Output did not include expected text. Output was:\n${result.output}` + ); + assert.ok( + result.output.includes("Exiting without installing malicious packages."), + `Output did not include expected text. Output was:\n${result.output}` + ); + }); +});