Improve e2e tests: add npm install tests, add test matrix

.github/workflows/test-on-pr.yml

@@ -40,6 +40,18 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        include:
+          - node_version: "lts"
+            npm_version: "latest"
+            yarn_version: "latest"
+            pnpm_version: "latest"
+          - node_version: "22"
+            npm_version: "10.0.0"
+            yarn_version: "latest"
+            pnpm_version: "latest"
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -53,6 +65,11 @@ jobs:
         run: npm ci
 
       - name: Run E2E tests
+        env:
+          NODE_VERSION: ${{ matrix.node_version }}
+          NPM_VERSION: ${{ matrix.npm_version }}
+          YARN_VERSION: ${{ matrix.yarn_version }}
+          PNPM_VERSION: ${{ matrix.pnpm_version }}
         run: npm run test:e2e
 
       - name: Clean up Docker resources

test/e2e/DockerTestContainer.js

@@ -1,14 +1,48 @@
 import { execSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
 import * as pty from "node-pty";
 import { parseShellOutput } from "./parseShellOutput.js";
 
+const imageName = "safe-chain-e2e-test";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const dockerFile = path.join(__dirname, "Dockerfile");
+const contextPath = path.join(__dirname, "../..");
+
+const nodeVersion = process.env.NODE_VERSION || "lts";
+const npmVersion = process.env.NPM_VERSION || "latest";
+const yarnVersion = process.env.YARN_VERSION || "latest";
+const pnpmVersion = process.env.PNPM_VERSION || "latest";
+
 export class DockerTestContainer {
-  constructor(imageName, containerName) {
-    this.imageName = imageName;
-    this.containerName = containerName;
+  constructor() {
+    this.containerName = `safe-chain-test-${Math.random()
+      .toString(36)
+      .substring(2, 15)}`;
     this.isRunning = false;
   }
 
+  static buildImage() {
+    try {
+      const buildArgs = [
+        `--build-arg NODE_VERSION=${nodeVersion}`,
+        `--build-arg NPM_VERSION=${npmVersion}`,
+        `--build-arg YARN_VERSION=${yarnVersion}`,
+        `--build-arg PNPM_VERSION=${pnpmVersion}`,
+      ].join(" ");
+
+      execSync(
+        `docker build -t ${imageName} -f ${dockerFile} ${contextPath} ${buildArgs}`,
+        {
+          stdio: "ignore",
+        }
+      );
+    } catch (error) {
+      throw new Error(`Failed to build Docker image: ${error.message}`);
+    }
+  }
+
   async start() {
     if (this.isRunning) {
       throw new Error("Container is already running");
@@ -17,7 +51,7 @@ export class DockerTestContainer {
     try {
       // Start a long-running container that we can exec commands into
       execSync(
-        `docker run -d --name ${this.containerName} ${this.imageName} sleep infinity`,
+        `docker run -d --name ${this.containerName} ${imageName} sleep infinity`,
         { stdio: "ignore" }
       );
       this.isRunning = true;

test/e2e/Dockerfile

@@ -18,15 +18,36 @@ COPY packages/safe-chain ./
 RUN npm --no-git-tag-version version 1.0.0 --allow-same-version
 RUN npm pack
 
-FROM mcr.microsoft.com/devcontainers/javascript-node:22-bookworm as runner
+FROM buildpack-deps:trixie
 
-WORKDIR /app
+# Package manager version arguments with defaults
+ARG NODE_VERSION=latest
+ARG NPM_VERSION=latest
+ARG YARN_VERSION=latest
+ARG PNPM_VERSION=latest
 
-COPY --from=builder /app/*.tgz /app/
+SHELL ["/bin/bash", "-c"]
+ENV BASH_ENV=~/.bashrc
 
-# # Install the application package globally
-RUN npm install -g /app/*.tgz
+# Install zsh
+RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/download/v1.2.1/zsh-in-docker.sh)"
+# Install fish
+RUN apt-get install -y fish && \
+    mkdir -p /root/.config/fish/ && \
+    touch /root/.config/fish/config.fish
 
-RUN mkdir /testapp
-RUN cd /testapp && npm init -y
+# Install Volta and Node.js
+RUN curl https://get.volta.sh | bash
+RUN volta install node@${NODE_VERSION}
+RUN volta install npm@${NPM_VERSION}
+RUN volta install yarn@${YARN_VERSION}
+RUN volta install pnpm@${PNPM_VERSION}
+
+# Copy and install Safe chain
+COPY --from=builder /app/*.tgz /pkgs/
+# RUN npm install -g /pkgs/*.tgz
+RUN npm install -g @aikidosec/safe-chain@1.0.21
+
+WORKDIR /testapp
+RUN npm init -y
 

test/e2e/npm.e2e.spec.js

@@ -0,0 +1,80 @@
+import { describe, it, before, beforeEach, afterEach } from "node:test";
+import { DockerTestContainer } from "./DockerTestContainer.js";
+import assert from "node:assert";
+
+describe("E2E: npm coverage", () => {
+  let container;
+
+  before(async () => {
+    DockerTestContainer.buildImage();
+  });
+
+  beforeEach(async () => {
+    // Run a new Docker container for each test
+    container = new DockerTestContainer();
+    await container.start();
+
+    const installationShell = await container.openShell("zsh");
+    await installationShell.runCommand("safe-chain setup");
+  });
+
+  afterEach(async () => {
+    // Stop and clean up the container after each test
+    if (container) {
+      await container.stop();
+      container = null;
+    }
+  });
+
+  it(`safe-chain succesfully installs safe packages`, async () => {
+    const shell = await container.openShell("zsh");
+    const result = await shell.runCommand("npm i axios");
+
+    assert.ok(
+      result.output.includes("No malicious packages detected."),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+  });
+
+  it(`safe-chain blocks installation of malicious packages`, async () => {
+    const shell = await container.openShell("zsh");
+    const result = await shell.runCommand("npm i safe-chain-test");
+
+    assert.ok(
+      result.output.includes("Malicious changes detected:"),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+    assert.ok(
+      result.output.includes("- safe-chain-test"),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+    assert.ok(
+      result.output.includes("Exiting without installing malicious packages."),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+
+    const listResult = await shell.runCommand("npm list");
+    assert.ok(
+      !listResult.output.includes("safe-chain-test"),
+      `Malicious package was installed despite safe-chain protection. Output of 'npm list' was:\n${listResult.output}`
+    );
+  });
+
+  it("safe-chain blocks npx from executing malicious packages", async () => {
+    const shell = await container.openShell("zsh");
+    const result = await shell.runCommand("npx safe-chain-test");
+
+    assert.ok(
+      result.output.includes("Malicious changes detected:"),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+    assert.ok(
+      result.output.includes("- safe-chain-test"),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+    assert.ok(
+      result.output.includes("Exiting without installing malicious packages."),
+      `Output did not include expected text. Output was:\n${result.output}`
+    );
+  });
+});

test/e2e/setup.teardown.e2e.spec.js

@@ -1,40 +1,20 @@
 import { describe, it, before, beforeEach, afterEach } from "node:test";
-import { execSync } from "node:child_process";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
 import { DockerTestContainer } from "./DockerTestContainer.js";
 import assert from "node:assert";
 
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
 describe("E2E: safe-chain setup command", () => {
-  const imageName = "safe-chain-e2e-test";
-  const containerName = "safe-chain-e2e-test-container";
   let container;
 
   before(async () => {
-    // Build the Docker image for the test environment
-    try {
-      const sourceDir = path.join(__dirname, "../..");
-      execSync(`docker build -t ${imageName} -f Dockerfile ${sourceDir}`, {
-        cwd: __dirname,
-        stdio: "ignore",
-      });
-    } catch (error) {
-      throw new Error(`Failed to setup test environment: ${error.message}`);
-    }
+    DockerTestContainer.buildImage();
   });
 
   beforeEach(async () => {
-    // Run a new Docker container for each test
-    container = new DockerTestContainer(imageName, containerName);
-
+    container = new DockerTestContainer();
     await container.start();
   });
 
   afterEach(async () => {
-    // Stop and clean up the container after each test
     if (container) {
       await container.stop();
       container = null;
@@ -51,9 +31,14 @@ describe("E2E: safe-chain setup command", () => {
       await projectShell.runCommand("cd /testapp");
       const result = await projectShell.runCommand("npm i axios");
 
+      const hasExpectedOutput = result.output.includes(
+        "Scanning for malicious packages..."
+      );
       assert.ok(
-        result.output.includes("Scanning for malicious packages..."),
-        "Expected npm command to be wrapped by safe-chain"
+        hasExpectedOutput,
+        hasExpectedOutput
+          ? "Expected npm command to be wrapped by safe-chain"
+          : `Output did not contain "Scanning for malicious packages...": \n${result.output}`
       );
     });