From c00abfb05457a8d70807585f2d14e2ee49daceb3 Mon Sep 17 00:00:00 2001 From: Sander Declerck Date: Fri, 18 Jul 2025 12:28:33 +0200 Subject: [PATCH] Add e2e tests --- .github/workflows/e2e.yml | 70 +++++++++++++++++++ e2e/aikido-npm.e2e.spec.js | 67 ++++++++++++++++++ e2e/aikido-npx.e2e.spec.js | 81 ++++++++++++++++++++++ e2e/aikido-pnpm.e2e.spec.js | 67 ++++++++++++++++++ e2e/aikido-pnpx.e2e.spec.js | 66 ++++++++++++++++++ e2e/aikido-yarn.e2e.spec.js | 70 +++++++++++++++++++ e2e/test-helpers.js | 133 ++++++++++++++++++++++++++++++++++++ package.json | 1 + 8 files changed, 555 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 e2e/aikido-npm.e2e.spec.js create mode 100644 e2e/aikido-npx.e2e.spec.js create mode 100644 e2e/aikido-pnpm.e2e.spec.js create mode 100644 e2e/aikido-pnpx.e2e.spec.js create mode 100644 e2e/aikido-yarn.e2e.spec.js create mode 100644 e2e/test-helpers.js diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..bdee5c4 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,70 @@ +name: E2E Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + e2e-tests: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node-version: [18, 20, 22] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: "npm" + + - name: Install dependencies + run: npm ci + + - name: Setup package managers (Ubuntu/macOS) + if: matrix.os != 'windows-latest' + run: | + # Install yarn + npm install -g yarn + + # Install pnpm + npm install -g pnpm + + # Verify installations + npm --version + yarn --version + pnpm --version + + - name: Setup package managers (Windows) + if: matrix.os == 'windows-latest' + run: | + # Install yarn + npm install -g yarn + + # Install pnpm + npm install -g pnpm + + # Verify installations + npm --version + yarn --version + pnpm --version + shell: pwsh + + - name: Install safe-chain globally + run: npm install -g . + + - name: Run unit tests + run: npm test + + - name: Run linting + run: npm run lint + + - name: Run E2E tests + run: npm run test:e2e diff --git a/e2e/aikido-npm.e2e.spec.js b/e2e/aikido-npm.e2e.spec.js new file mode 100644 index 0000000..6729e7e --- /dev/null +++ b/e2e/aikido-npm.e2e.spec.js @@ -0,0 +1,67 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { createTempDir, cleanupTempDir, runAikidoCommand, isPackageManagerAvailable } from './test-helpers.js'; + +describe('aikido-npm e2e tests', () => { + let tempDir; + + beforeEach(async () => { + tempDir = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(tempDir); + }); + + it('should allow installation of legitimate package (axios)', async () => { + // Fail if npm is not available + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npm', ['install', 'axios', '--dry-run'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should succeed (exit code 0) and not show malware warning + assert.equal(result.code, 0, `Expected success but got: ${result.stderr}`); + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect axios as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect axios as malware'); + }); + + it('should block installation of malware package (eslint-js)', async () => { + // Fail if npm is not available + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npm', ['install', 'eslint-js'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should fail (non-zero exit code) and show malware warning + assert.notEqual(result.code, 0, 'Should fail when trying to install malware'); + + // Check that malware was detected + const output = result.stdout + result.stderr; + assert.ok( + output.includes('malware') || output.includes('MALWARE') || output.includes('blocked') || output.includes('dangerous') || output.includes('Malicious changes detected'), + `Should detect malware but got: ${output}` + ); + }); + + it('should handle npm install with version specifiers', async () => { + // Fail if npm is not available + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npm', ['install', 'axios@1.0.0', '--dry-run'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should succeed with version specifier + assert.equal(result.code, 0, `Expected success with version specifier but got: ${result.stderr}`); + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect axios with version as malware'); + }); +}); \ No newline at end of file diff --git a/e2e/aikido-npx.e2e.spec.js b/e2e/aikido-npx.e2e.spec.js new file mode 100644 index 0000000..71cd4f8 --- /dev/null +++ b/e2e/aikido-npx.e2e.spec.js @@ -0,0 +1,81 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { createTempDir, cleanupTempDir, runAikidoCommand, isPackageManagerAvailable } from './test-helpers.js'; + +describe('aikido-npx e2e tests', () => { + let tempDir; + + beforeEach(async () => { + tempDir = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(tempDir); + }); + + it('should allow execution of legitimate package (cowsay)', async () => { + // Fail if npm is not available (npx comes with npm) + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm/npx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npx', ['cowsay', '--help'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should not detect cowsay as malware, regardless of execution result + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect cowsay as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect cowsay as malware'); + }); + + it('should block execution of malware package (eslint-js)', async () => { + // Fail if npm is not available (npx comes with npm) + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm/npx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npx', ['eslint-js'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should fail (non-zero exit code) and show malware warning + assert.notEqual(result.code, 0, 'Should fail when trying to execute malware'); + + // Check that malware was detected + const output = result.stdout + result.stderr; + assert.ok( + output.includes('malware') || output.includes('MALWARE') || output.includes('blocked') || output.includes('dangerous') || output.includes('Malicious changes detected'), + `Should detect malware but got: ${output}` + ); + }); + + it('should handle npx with version specifiers', async () => { + // Fail if npm is not available (npx comes with npm) + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm/npx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npx', ['cowsay@1.0.0', '--help'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should not detect cowsay with version as malware + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect cowsay with version as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect cowsay with version as malware'); + }); + + it('should handle npx with package arguments', async () => { + // Fail if npm is not available (npx comes with npm) + const npmAvailable = await isPackageManagerAvailable('npm'); + assert.ok(npmAvailable, 'npm/npx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-npx', ['cowsay', 'hello world'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should not detect cowsay as malware, regardless of execution result + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect cowsay as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect cowsay as malware'); + }); +}); \ No newline at end of file diff --git a/e2e/aikido-pnpm.e2e.spec.js b/e2e/aikido-pnpm.e2e.spec.js new file mode 100644 index 0000000..c0262aa --- /dev/null +++ b/e2e/aikido-pnpm.e2e.spec.js @@ -0,0 +1,67 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { createTempDir, cleanupTempDir, runAikidoCommand, isPackageManagerAvailable } from './test-helpers.js'; + +describe('aikido-pnpm e2e tests', () => { + let tempDir; + + beforeEach(async () => { + tempDir = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(tempDir); + }); + + it('should allow installation of legitimate package (axios)', async () => { + // Fail if pnpm is not available + const pnpmAvailable = await isPackageManagerAvailable('pnpm'); + assert.ok(pnpmAvailable, 'pnpm is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-pnpm', ['add', 'axios'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should succeed (exit code 0) and not show malware warning + // Note: pnpm may still exit with non-zero due to network issues, but should not show malware warnings + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect axios as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect axios as malware'); + }); + + it('should block installation of malware package (eslint-js)', async () => { + // Fail if pnpm is not available + const pnpmAvailable = await isPackageManagerAvailable('pnpm'); + assert.ok(pnpmAvailable, 'pnpm is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-pnpm', ['add', 'eslint-js'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should fail (non-zero exit code) and show malware warning + assert.notEqual(result.code, 0, 'Should fail when trying to install malware'); + + // Check that malware was detected + const output = result.stdout + result.stderr; + assert.ok( + output.includes('malware') || output.includes('MALWARE') || output.includes('blocked') || output.includes('dangerous') || output.includes('Malicious changes detected'), + `Should detect malware but got: ${output}` + ); + }); + + it('should handle pnpm add with version specifiers', async () => { + // Fail if pnpm is not available + const pnpmAvailable = await isPackageManagerAvailable('pnpm'); + assert.ok(pnpmAvailable, 'pnpm is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-pnpm', ['add', 'axios@1.0.0'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should succeed with version specifier + // Note: pnpm may still exit with non-zero due to network issues, but should not show malware warnings + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect axios with version as malware'); + }); +}); \ No newline at end of file diff --git a/e2e/aikido-pnpx.e2e.spec.js b/e2e/aikido-pnpx.e2e.spec.js new file mode 100644 index 0000000..70208af --- /dev/null +++ b/e2e/aikido-pnpx.e2e.spec.js @@ -0,0 +1,66 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { createTempDir, cleanupTempDir, runAikidoCommand, isPackageManagerAvailable } from './test-helpers.js'; + +describe('aikido-pnpx e2e tests', () => { + let tempDir; + + beforeEach(async () => { + tempDir = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(tempDir); + }); + + it('should allow execution of legitimate package (cowsay)', async () => { + // Fail if pnpm is not available (pnpx comes with pnpm) + const pnpmAvailable = await isPackageManagerAvailable('pnpm'); + assert.ok(pnpmAvailable, 'pnpm/pnpx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-pnpx', ['cowsay', '--help'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should not detect cowsay as malware, regardless of execution result + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect cowsay as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect cowsay as malware'); + }); + + it('should block execution of malware package (eslint-js)', async () => { + // Fail if pnpm is not available (pnpx comes with pnpm) + const pnpmAvailable = await isPackageManagerAvailable('pnpm'); + assert.ok(pnpmAvailable, 'pnpm/pnpx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-pnpx', ['eslint-js'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should fail (non-zero exit code) and show malware warning + assert.notEqual(result.code, 0, 'Should fail when trying to execute malware'); + + // Check that malware was detected + const output = result.stdout + result.stderr; + assert.ok( + output.includes('malware') || output.includes('MALWARE') || output.includes('blocked') || output.includes('dangerous') || output.includes('Malicious changes detected'), + `Should detect malware but got: ${output}` + ); + }); + + it('should handle pnpx with version specifiers', async () => { + // Fail if pnpm is not available (pnpx comes with pnpm) + const pnpmAvailable = await isPackageManagerAvailable('pnpm'); + assert.ok(pnpmAvailable, 'pnpm/pnpx is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-pnpx', ['cowsay@1.0.0', '--help'], { + cwd: tempDir, + timeout: 10000 + }); + + // Should not detect cowsay with version as malware + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect cowsay with version as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect cowsay with version as malware'); + }); +}); \ No newline at end of file diff --git a/e2e/aikido-yarn.e2e.spec.js b/e2e/aikido-yarn.e2e.spec.js new file mode 100644 index 0000000..0c9c8c4 --- /dev/null +++ b/e2e/aikido-yarn.e2e.spec.js @@ -0,0 +1,70 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { createTempDir, cleanupTempDir, runAikidoCommand, isPackageManagerAvailable } from './test-helpers.js'; + +describe('aikido-yarn e2e tests', () => { + let tempDir; + + beforeEach(async () => { + tempDir = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(tempDir); + }); + + it('should allow installation of legitimate package (axios)', async () => { + // Fail if yarn is not available + const yarnAvailable = await isPackageManagerAvailable('yarn'); + assert.ok(yarnAvailable, 'yarn is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-yarn', ['add', 'axios', '--dry-run'], { + cwd: tempDir, + timeout: 10000, + env: { NPM_TOKEN: 'test-token' } // Set NPM_TOKEN to avoid yarn config error + }); + + // Should succeed (exit code 0) and not show malware warning + assert.equal(result.code, 0, `Expected success but got: ${result.stderr}`); + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect axios as malware'); + assert.ok(!result.stderr.includes('MALWARE'), 'Should not detect axios as malware'); + }); + + it('should block installation of malware package (eslint-js)', async () => { + // Fail if yarn is not available + const yarnAvailable = await isPackageManagerAvailable('yarn'); + assert.ok(yarnAvailable, 'yarn is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-yarn', ['add', 'eslint-js'], { + cwd: tempDir, + timeout: 10000, + env: { NPM_TOKEN: 'test-token' } // Set NPM_TOKEN to avoid yarn config error + }); + + // Should fail (non-zero exit code) and show malware warning + assert.notEqual(result.code, 0, 'Should fail when trying to install malware'); + + // Check that malware was detected + const output = result.stdout + result.stderr; + assert.ok( + output.includes('malware') || output.includes('MALWARE') || output.includes('blocked') || output.includes('dangerous') || output.includes('Malicious changes detected'), + `Should detect malware but got: ${output}` + ); + }); + + it('should handle yarn add with version specifiers', async () => { + // Fail if yarn is not available + const yarnAvailable = await isPackageManagerAvailable('yarn'); + assert.ok(yarnAvailable, 'yarn is not available - check CI/CD configuration'); + + const result = await runAikidoCommand('aikido-yarn', ['add', 'axios@1.0.0', '--dry-run'], { + cwd: tempDir, + timeout: 10000, + env: { NPM_TOKEN: 'test-token' } // Set NPM_TOKEN to avoid yarn config error + }); + + // Should succeed with version specifier + assert.equal(result.code, 0, `Expected success with version specifier but got: ${result.stderr}`); + assert.ok(!result.stdout.includes('MALWARE'), 'Should not detect axios with version as malware'); + }); +}); \ No newline at end of file diff --git a/e2e/test-helpers.js b/e2e/test-helpers.js new file mode 100644 index 0000000..a80e308 --- /dev/null +++ b/e2e/test-helpers.js @@ -0,0 +1,133 @@ +import { spawn } from 'child_process'; +import { mkdtemp, rm } from 'fs/promises'; +import { join } from 'path'; +import { tmpdir } from 'os'; + +/** + * Creates a temporary directory for testing + */ +export async function createTempDir() { + const { writeFile } = await import('fs/promises'); + const tempDir = await mkdtemp(join(tmpdir(), 'aikido-e2e-')); + + // Create a basic package.json to avoid yarn/pnpm issues + const packageJson = { + name: 'test-project', + version: '1.0.0', + description: 'Test project for e2e tests' + }; + + await writeFile(join(tempDir, 'package.json'), JSON.stringify(packageJson, null, 2)); + + return tempDir; +} + +/** + * Cleans up a temporary directory + */ +export async function cleanupTempDir(tempDir) { + try { + await rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } +} + +/** + * Runs a command and captures stdout/stderr + */ +export function runCommand(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + stdio: 'pipe', + ...options + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('close', (code) => { + resolve({ + code, + stdout, + stderr + }); + }); + + child.on('error', (error) => { + reject(error); + }); + }); +} + +/** + * Runs an aikido command with timeout + */ +export async function runAikidoCommand(binaryName, args, options = {}) { + const binaryPath = join(process.cwd(), 'bin', `${binaryName}.js`); + const timeout = options.timeout || 10000; // 10 second timeout + + return new Promise((resolve, reject) => { + const child = spawn('node', [binaryPath, ...args], { + stdio: 'pipe', + cwd: options.cwd || process.cwd(), + env: { ...process.env, ...options.env } + }); + + let stdout = ''; + let stderr = ''; + let timeoutId; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('close', (code) => { + clearTimeout(timeoutId); + resolve({ + code, + stdout, + stderr + }); + }); + + child.on('error', (error) => { + clearTimeout(timeoutId); + reject(error); + }); + + // Set timeout + timeoutId = setTimeout(() => { + child.kill('SIGKILL'); + resolve({ + code: 1, + stdout, + stderr: stderr + '\n[Test timeout - process killed]' + }); + }, timeout); + }); +} + +/** + * Checks if a package manager is available in the system + */ +export async function isPackageManagerAvailable(packageManager) { + try { + const result = await runCommand(packageManager, ['--version']); + return result.code === 0; + } catch { + return false; + } +} \ No newline at end of file diff --git a/package.json b/package.json index 4ab4c95..cd530c6 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "scripts": { "test": "node --test --experimental-test-module-mocks **/*.spec.js", "test:watch": "node --test --watch --experimental-test-module-mocks **/*.spec.js", + "test:e2e": "node --test e2e/**/*.spec.js", "lint": "eslint ." }, "repository": {