Merge pull request #365 from 123Haynes/main

add a configuration option for custom malwaredb and newpackagelist urls.

README.md

@@ -281,6 +281,41 @@ You can set custom registries through environment variable or config file. Both
    }
    ```
 
+## Malware List Base URL
+
+Configure Safe Chain to fetch malware databases and new packages lists from a custom mirror URL. This allows you to host your own copy of the Aikido malware database.
+
+### Configuration Options
+
+You can set the malware list base URL through multiple sources (in order of priority):
+
+1. **CLI Argument** (highest priority):
+
+   ```shell
+   npm install express --safe-chain-malware-list-base-url=https://your-mirror.com
+   ```
+
+2. **Environment Variable**:
+
+   ```shell
+   export SAFE_CHAIN_MALWARE_LIST_BASE_URL=https://your-mirror.com
+   npm install express
+   ```
+
+3. **Config File** (`~/.safe-chain/config.json`):
+
+   ```json
+   {
+     "malwareListBaseUrl": "https://your-mirror.com"
+   }
+   ```
+
+The base URL should point to a server that mirrors the structure of `https://malware-list.aikido.dev/`, including the following paths:
+- `/malware_predictions.json` (JavaScript ecosystem malware database)
+- `/malware_pypi.json` (Python ecosystem malware database)
+- `/releases/npm.json` (JavaScript new packages list)
+- `/releases/pypi.json` (Python new packages list)
+
 # Usage in CI/CD
 
 You can protect your CI/CD pipelines from malicious packages by integrating Aikido Safe Chain into your build process. This ensures that any packages installed during your automated builds are checked for malware before installation.

packages/safe-chain/src/api/aikido.js

@@ -3,17 +3,18 @@ import {
   getEcoSystem,
   ECOSYSTEM_JS,
   ECOSYSTEM_PY,
+  getMalwareListBaseUrl,
 } from "../config/settings.js";
 import { ui } from "../environment/userInteraction.js";
 
-const malwareDatabaseUrls = {
-  [ECOSYSTEM_JS]: "https://malware-list.aikido.dev/malware_predictions.json",
-  [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/malware_pypi.json",
+const malwareDatabasePaths = {
+  [ECOSYSTEM_JS]: "malware_predictions.json",
+  [ECOSYSTEM_PY]: "malware_pypi.json",
 };
 
-const newPackagesListUrls = {
-  [ECOSYSTEM_JS]: "https://malware-list.aikido.dev/releases/npm.json",
-  [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/releases/pypi.json",
+const newPackagesListPaths = {
+  [ECOSYSTEM_JS]: "releases/npm.json",
+  [ECOSYSTEM_PY]: "releases/pypi.json",
 };
 
 const DEFAULT_FETCH_RETRY_ATTEMPTS = 4;
@@ -40,10 +41,11 @@ const DEFAULT_FETCH_RETRY_ATTEMPTS = 4;
 export async function fetchMalwareDatabase() {
   return retry(async () => {
     const ecosystem = getEcoSystem();
-    const malwareDatabaseUrl =
-      malwareDatabaseUrls[
-        /** @type {keyof typeof malwareDatabaseUrls} */ (ecosystem)
+    const baseUrl = getMalwareListBaseUrl();
+    const path = malwareDatabasePaths[
+      /** @type {keyof typeof malwareDatabasePaths} */ (ecosystem)
     ];
+    const malwareDatabaseUrl = `${baseUrl}/${path}`;
     const response = await fetch(malwareDatabaseUrl);
     if (!response.ok) {
       throw new Error(
@@ -69,10 +71,11 @@ export async function fetchMalwareDatabase() {
 export async function fetchMalwareDatabaseVersion() {
   return retry(async () => {
     const ecosystem = getEcoSystem();
-    const malwareDatabaseUrl =
-      malwareDatabaseUrls[
-        /** @type {keyof typeof malwareDatabaseUrls} */ (ecosystem)
+    const baseUrl = getMalwareListBaseUrl();
+    const path = malwareDatabasePaths[
+      /** @type {keyof typeof malwareDatabasePaths} */ (ecosystem)
     ];
+    const malwareDatabaseUrl = `${baseUrl}/${path}`;
     const response = await fetch(malwareDatabaseUrl, {
       method: "HEAD",
     });
@@ -92,13 +95,15 @@ export async function fetchMalwareDatabaseVersion() {
 export async function fetchNewPackagesList() {
   return retry(async () => {
     const ecosystem = getEcoSystem();
-    const url =
-      newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)];
+    const baseUrl = getMalwareListBaseUrl();
+    const path = newPackagesListPaths[/** @type {keyof typeof newPackagesListPaths} */ (ecosystem)];
 
-    if (!url) {
+    if (!path) {
       return { newPackagesList: [], version: undefined };
     }
 
+    const url = `${baseUrl}/${path}`;
+
     const response = await fetch(url);
     if (!response.ok) {
       throw new Error(
@@ -124,13 +129,15 @@ export async function fetchNewPackagesList() {
 export async function fetchNewPackagesListVersion() {
   return retry(async () => {
     const ecosystem = getEcoSystem();
-    const url =
-      newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)];
+    const baseUrl = getMalwareListBaseUrl();
+    const path = newPackagesListPaths[/** @type {keyof typeof newPackagesListPaths} */ (ecosystem)];
 
-    if (!url) {
+    if (!path) {
       return undefined;
     }
 
+    const url = `${baseUrl}/${path}`;
+
     const response = await fetch(url, { method: "HEAD" });
     if (!response.ok) {
       throw new Error(

packages/safe-chain/src/api/aikido.spec.js

@@ -22,6 +22,7 @@ describe("aikido API", async () => {
       getEcoSystem: () => ecosystem,
       ECOSYSTEM_JS: "js",
       ECOSYSTEM_PY: "py",
+      getMalwareListBaseUrl: () => "https://malware-list.aikido.dev",
     },
   });
 
@@ -184,6 +185,15 @@ describe("aikido API", async () => {
       assert.deepStrictEqual(result.newPackagesList, []);
       assert.strictEqual(result.version, undefined);
     });
+
+    it("should return undefined version without fetching for unsupported ecosystems", async () => {
+      ecosystem = "ruby";
+
+      const result = await fetchNewPackagesListVersion();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 0);
+      assert.strictEqual(result, undefined);
+    });
   });
 
   describe("fetchNewPackagesListVersion", () => {

packages/safe-chain/src/config/cliArguments.js

@@ -1,12 +1,13 @@
 import { ui } from "../environment/userInteraction.js";
 
 /**
- * @type {{loggingLevel: string | undefined, skipMinimumPackageAge: boolean | undefined, minimumPackageAgeHours: string | undefined}}
+ * @type {{loggingLevel: string | undefined, skipMinimumPackageAge: boolean | undefined, minimumPackageAgeHours: string | undefined, malwareListBaseUrl: string | undefined}}
  */
 const state = {
   loggingLevel: undefined,
   skipMinimumPackageAge: undefined,
   minimumPackageAgeHours: undefined,
+  malwareListBaseUrl: undefined,
 };
 
 const SAFE_CHAIN_ARG_PREFIX = "--safe-chain-";
@@ -20,6 +21,7 @@ export function initializeCliArguments(args) {
   state.loggingLevel = undefined;
   state.skipMinimumPackageAge = undefined;
   state.minimumPackageAgeHours = undefined;
+  state.malwareListBaseUrl = undefined;
 
   const safeChainArgs = [];
   const remainingArgs = [];
@@ -35,6 +37,7 @@ export function initializeCliArguments(args) {
   setLoggingLevel(safeChainArgs);
   setSkipMinimumPackageAge(safeChainArgs);
   setMinimumPackageAgeHours(safeChainArgs);
+  setMalwareListBaseUrl(safeChainArgs);
   checkDeprecatedPythonFlag(args);
   return remainingArgs;
 }
@@ -109,6 +112,26 @@ export function getMinimumPackageAgeHours() {
   return state.minimumPackageAgeHours;
 }
 
+/**
+ * @param {string[]} args
+ * @returns {void}
+ */
+function setMalwareListBaseUrl(args) {
+  const argName = SAFE_CHAIN_ARG_PREFIX + "malware-list-base-url=";
+
+  const value = getLastArgEqualsValue(args, argName);
+  if (value) {
+    state.malwareListBaseUrl = value;
+  }
+}
+
+/**
+ * @returns {string | undefined}
+ */
+export function getMalwareListBaseUrl() {
+  return state.malwareListBaseUrl;
+}
+
 /**
  * @param {string[]} args
  * @param {string} flagName

packages/safe-chain/src/config/configFile.js

@@ -10,6 +10,7 @@ import { getEcoSystem } from "./settings.js";
  * We cannot trust the input and should add the necessary validations
  * @property {unknown | Number} scanTimeout
  * @property {unknown | Number} minimumPackageAgeHours
+ * @property {unknown | string} malwareListBaseUrl
  * @property {unknown | SafeChainRegistryConfiguration} npm
  * @property {unknown | SafeChainRegistryConfiguration} pip
  *
@@ -84,6 +85,18 @@ export function getMinimumPackageAgeHours() {
   return undefined;
 }
 
+/**
+ * Gets the malware list base URL from config file only
+ * @returns {string | undefined}
+ */
+export function getMalwareListBaseUrl() {
+  const config = readConfigFile();
+  if (config.malwareListBaseUrl && typeof config.malwareListBaseUrl === "string") {
+    return config.malwareListBaseUrl;
+  }
+  return undefined;
+}
+
 /**
  * Gets the custom npm registries from the config file (format parsing only, no validation)
  * @returns {string[]}
@@ -214,6 +227,7 @@ function readConfigFile() {
   const emptyConfig = {
     scanTimeout: undefined,
     minimumPackageAgeHours: undefined,
+    malwareListBaseUrl: undefined,
     npm: {
       customRegistries: undefined,
     },

packages/safe-chain/src/config/environmentVariables.js

@@ -45,3 +45,13 @@ export function getMinimumPackageAgeExclusions() {
   return process.env.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS ||
     process.env.SAFE_CHAIN_NPM_MINIMUM_PACKAGE_AGE_EXCLUSIONS;
 }
+
+/**
+ * Gets the malware list base URL from environment variable
+ * Expected format: full URL without trailing slash
+ * Example: "https://malware-list.aikido.dev"
+ * @returns {string | undefined}
+ */
+export function getMalwareListBaseUrl() {
+  return process.env.SAFE_CHAIN_MALWARE_LIST_BASE_URL;
+}

packages/safe-chain/src/config/settings.js

@@ -1,6 +1,7 @@
 import * as cliArguments from "./cliArguments.js";
 import * as configFile from "./configFile.js";
 import * as environmentVariables from "./environmentVariables.js";
+import { ui } from "../environment/userInteraction.js";
 
 export const LOGGING_SILENT = "silent";
 export const LOGGING_NORMAL = "normal";
@@ -198,3 +199,51 @@ export function getMinimumPackageAgeExclusions() {
   const allExclusions = [...envExclusions, ...configExclusions];
   return [...new Set(allExclusions)];
 }
+
+/**
+ * Gets the malware list base URL with priority: CLI argument > environment variable > config file > default
+ * @returns {string}
+ */
+export function getMalwareListBaseUrl() {
+  // Priority 1: CLI argument
+  const cliValue = cliArguments.getMalwareListBaseUrl();
+  if (cliValue) {
+    const url = removeTrailingSlashes(cliValue);
+    ui.writeInformation(`Fetching malware lists from ${url} as defined by CLI argument --safe-chain-malware-list-base-url`);
+    return url;
+  }
+
+  // Priority 2: Environment variable
+  const envValue = environmentVariables.getMalwareListBaseUrl();
+  if (envValue) {
+    const url = removeTrailingSlashes(envValue);
+    ui.writeInformation(`Fetching malware lists from ${url} as defined by environment variable SAFE_CHAIN_MALWARE_LIST_BASE_URL`);
+    return url;
+  }
+
+  // Priority 3: Config file
+  const configValue = configFile.getMalwareListBaseUrl();
+  if (configValue) {
+    const url = removeTrailingSlashes(configValue);
+    ui.writeInformation(`Fetching malware lists from ${url} as defined by config file (malwareListBaseUrl)`);
+    return url;
+  }
+
+  // Default
+  const url = removeTrailingSlashes("https://malware-list.aikido.dev");
+  ui.writeInformation(`Fetching malware lists from ${url} (default)`);
+  return url;
+}
+
+/**
+ * Removes trailing slashes from a URL-like string.
+ * @param {string} value
+ * @returns {string}
+ */
+function removeTrailingSlashes(value) {
+  if (!value || typeof value !== "string") {
+    return value;
+  }
+
+  return value.replace(/\/+$/, "");
+}

packages/safe-chain/src/config/settings.spec.js

@@ -15,6 +15,7 @@ const {
   getNpmCustomRegistries,
   getPipCustomRegistries,
   getMinimumPackageAgeExclusions,
+  getMalwareListBaseUrl,
   setEcoSystem,
   ECOSYSTEM_JS,
   ECOSYSTEM_PY,
@@ -534,3 +535,113 @@ describe("getMinimumPackageAgeExclusions", () => {
     assert.deepStrictEqual(exclusions, ["requests", "urllib3"]);
   });
 });
+
+describe("getMalwareListBaseUrl", () => {
+  let originalEnv;
+  const envVarName = "SAFE_CHAIN_MALWARE_LIST_BASE_URL";
+
+  beforeEach(() => {
+    originalEnv = process.env[envVarName];
+    delete process.env[envVarName];
+    // Reset CLI arguments state
+    initializeCliArguments([]);
+  });
+
+  afterEach(() => {
+    if (originalEnv !== undefined) {
+      process.env[envVarName] = originalEnv;
+    } else {
+      delete process.env[envVarName];
+    }
+    configFileContent = undefined;
+  });
+
+  it("should return default URL when nothing is configured", () => {
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://malware-list.aikido.dev");
+  });
+
+  it("should trim trailing slash from CLI argument", () => {
+    initializeCliArguments(["--safe-chain-malware-list-base-url=https://cli-mirror.com/"]);
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://cli-mirror.com");
+  });
+
+  it("should trim trailing slash from environment variable", () => {
+    process.env[envVarName] = "https://env-mirror.com/";
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://env-mirror.com");
+  });
+
+  it("should trim trailing slash from config file value", () => {
+    configFileContent = JSON.stringify({
+      malwareListBaseUrl: "https://config-mirror.com/",
+    });
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://config-mirror.com");
+  });
+
+  it("should return CLI argument value with highest priority", () => {
+    initializeCliArguments(["--safe-chain-malware-list-base-url=https://cli-mirror.com"]);
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://cli-mirror.com");
+  });
+
+  it("should return environment variable value when no CLI argument", () => {
+    process.env[envVarName] = "https://env-mirror.com";
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://env-mirror.com");
+  });
+
+  it("should return config file value when no CLI or env", () => {
+    configFileContent = JSON.stringify({
+      malwareListBaseUrl: "https://config-mirror.com",
+    });
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://config-mirror.com");
+  });
+
+  it("should prioritize CLI over environment variable", () => {
+    process.env[envVarName] = "https://env-mirror.com";
+    initializeCliArguments(["--safe-chain-malware-list-base-url=https://cli-mirror.com"]);
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://cli-mirror.com");
+  });
+
+  it("should prioritize environment variable over config file", () => {
+    process.env[envVarName] = "https://env-mirror.com";
+    configFileContent = JSON.stringify({
+      malwareListBaseUrl: "https://config-mirror.com",
+    });
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://env-mirror.com");
+  });
+
+  it("should prioritize CLI over config file", () => {
+    initializeCliArguments(["--safe-chain-malware-list-base-url=https://cli-mirror.com"]);
+    configFileContent = JSON.stringify({
+      malwareListBaseUrl: "https://config-mirror.com",
+    });
+
+    const url = getMalwareListBaseUrl();
+
+    assert.strictEqual(url, "https://cli-mirror.com");
+  });
+});

packages/safe-chain/src/scanning/newPackagesDatabase.spec.js

@@ -51,6 +51,7 @@ mock.module("../config/settings.js", {
   namedExports: {
     getMinimumPackageAgeHours: () => minimumPackageAgeHours,
     getEcoSystem: () => ecosystem,
+    getMalwareListBaseUrl: () => "https://malware-list.aikido.dev",
     ECOSYSTEM_JS: "js",
     ECOSYSTEM_PY: "py",
   },

packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.spec.js

@@ -8,6 +8,7 @@ mock.module("../config/settings.js", {
   namedExports: {
     getMinimumPackageAgeHours: () => minimumPackageAgeHours,
     getEcoSystem: () => ecosystem,
+    getMalwareListBaseUrl: () => "https://malware-list.aikido.dev",
     ECOSYSTEM_JS: "js",
     ECOSYSTEM_PY: "py",
   },

packages/safe-chain/src/scanning/newPackagesListCache.spec.js

@@ -20,6 +20,7 @@ mock.module("../config/settings.js", {
   namedExports: {
     getEcoSystem: () => ecosystem,
     getMinimumPackageAgeHours: () => 24,
+    getMalwareListBaseUrl: () => "https://malware-list.aikido.dev",
     ECOSYSTEM_JS: "js",
     ECOSYSTEM_PY: "py",
   },