Merge pull request #356 from AikidoSec/split-up-new-packages-database

Split up newPackagesDatabse into builder, warnigns, cache

packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js

@@ -13,7 +13,7 @@ import {
   modifyNpmInfoResponse,
 } from "./modifyNpmInfo.js";
 import { parseNpmPackageUrl } from "./parseNpmPackageUrl.js";
-import { openNewPackagesDatabase } from "../../../scanning/newPackagesDatabase.js";
+import { openNewPackagesDatabase } from "../../../scanning/newPackagesListCache.js";
 
 const knownJsRegistries = [
   "registry.npmjs.org",

packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js

@@ -18,7 +18,7 @@ describe("npmInterceptor minimum package age", async () => {
       getEcoSystem: () => "js",
     },
   });
-  mock.module("../../../scanning/newPackagesDatabase.js", {
+  mock.module("../../../scanning/newPackagesListCache.js", {
     namedExports: {
       openNewPackagesDatabase: async () => ({
         isNewlyReleasedPackage: (name, version) =>

packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.packageDownload.spec.js

@@ -32,7 +32,7 @@ mock.module("../../../config/settings.js", {
     skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
   },
 });
-mock.module("../../../scanning/newPackagesDatabase.js", {
+mock.module("../../../scanning/newPackagesListCache.js", {
   namedExports: {
     openNewPackagesDatabase: async () => ({
       isNewlyReleasedPackage: (name, version) =>

packages/safe-chain/src/scanning/newPackagesDatabase.spec.js

@@ -56,6 +56,9 @@ mock.module("../config/settings.js", {
   },
 });
 
+// Import the warnings module so we can reset its state between tests.
+const { resetWarningState } = await import("./newPackagesDatabaseWarnings.js");
+
 describe("newPackagesDatabase", async () => {
   beforeEach(() => {
     fetchedList = [];
@@ -66,6 +69,7 @@ describe("newPackagesDatabase", async () => {
     writeWarningCalls = [];
     fetchListError = null;
     fetchVersionError = null;
+    resetWarningState();
     testHomeDir = path.join(
       os.tmpdir(),
       `safe-chain-new-packages-db-${process.pid}-${importCounter}`
@@ -77,13 +81,13 @@ describe("newPackagesDatabase", async () => {
 
   async function openNewPackagesDatabase() {
     const module = await import(
-      `./newPackagesDatabase.js?test_case=${importCounter++}`
+      `./newPackagesListCache.js?test_case=${importCounter++}`
     );
     return module.openNewPackagesDatabase();
   }
 
   async function loadNewPackagesDatabaseModule() {
-    return import(`./newPackagesDatabase.js?test_case=${importCounter++}`);
+    return import(`./newPackagesListCache.js?test_case=${importCounter++}`);
   }
 
   function hoursAgo(hours) {

packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.js

@@ -0,0 +1,63 @@
+import {
+  getMinimumPackageAgeHours,
+  getEcoSystem,
+  ECOSYSTEM_JS,
+  ECOSYSTEM_PY,
+} from "../config/settings.js";
+
+/**
+ * @typedef {Object} NewPackagesDatabase
+ * @property {function(string, string): boolean} isNewlyReleasedPackage
+ */
+
+/**
+ * Returns the ecosystem identifier expected in upstream/core release feeds.
+ * @returns {string}
+ */
+function getCurrentFeedSource() {
+  const ecosystem = getEcoSystem();
+
+  if (ecosystem === ECOSYSTEM_JS) {
+    return "npm";
+  }
+
+  if (ecosystem === ECOSYSTEM_PY) {
+    return "pypi";
+  }
+
+  return ecosystem;
+}
+
+/**
+ * @param {import("../api/aikido.js").NewPackageEntry[]} newPackagesList
+ * @returns {NewPackagesDatabase}
+ */
+export function buildNewPackagesDatabase(newPackagesList) {
+  /**
+   * @param {string} name
+   * @param {string} version
+   * @returns {boolean}
+   */
+  function isNewlyReleasedPackage(name, version) {
+    const cutOff = new Date(
+      new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
+    );
+    const expectedSource = getCurrentFeedSource();
+
+    const entry = newPackagesList.find(
+      (pkg) =>
+        (!pkg.source || pkg.source.toLowerCase() === expectedSource) &&
+        pkg.package_name === name &&
+        pkg.version === version
+    );
+
+    if (!entry) {
+      return false;
+    }
+
+    const releasedOn = new Date(entry.released_on * 1000);
+    return releasedOn > cutOff;
+  }
+
+  return { isNewlyReleasedPackage };
+}

packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.spec.js

@@ -0,0 +1,100 @@
+import { describe, it, mock } from "node:test";
+import assert from "node:assert";
+
+let minimumPackageAgeHours = 24;
+let ecosystem = "js";
+
+mock.module("../config/settings.js", {
+  namedExports: {
+    getMinimumPackageAgeHours: () => minimumPackageAgeHours,
+    getEcoSystem: () => ecosystem,
+    ECOSYSTEM_JS: "js",
+    ECOSYSTEM_PY: "py",
+  },
+});
+
+const { buildNewPackagesDatabase } = await import(
+  "./newPackagesDatabaseBuilder.js"
+);
+
+function hoursAgo(hours) {
+  return Math.floor((Date.now() - hours * 3600 * 1000) / 1000);
+}
+
+describe("buildNewPackagesDatabase", () => {
+  it("returns an object with isNewlyReleasedPackage", () => {
+    const db = buildNewPackagesDatabase([]);
+    assert.strictEqual(typeof db.isNewlyReleasedPackage, "function");
+  });
+
+  describe("isNewlyReleasedPackage", () => {
+    it("returns true for a package released within the age threshold", () => {
+      const db = buildNewPackagesDatabase([
+        { package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) },
+      ]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("returns false for a package released outside the age threshold", () => {
+      const db = buildNewPackagesDatabase([
+        { package_name: "foo", version: "1.0.0", released_on: hoursAgo(48) },
+      ]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+
+    it("returns false for a package not in the list", () => {
+      const db = buildNewPackagesDatabase([]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("not-there", "1.0.0"), false);
+    });
+
+    it("returns false for a known package but different version", () => {
+      const db = buildNewPackagesDatabase([
+        { package_name: "foo", version: "2.0.0", released_on: hoursAgo(1) },
+      ]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+
+    it("filters by source when source metadata is present", () => {
+      const db = buildNewPackagesDatabase([
+        { source: "pypi", package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) },
+        { source: "npm", package_name: "bar", version: "1.0.0", released_on: hoursAgo(1) },
+      ]);
+
+      // ecosystem is "js" → feed source is "npm"
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+      assert.strictEqual(db.isNewlyReleasedPackage("bar", "1.0.0"), true);
+    });
+
+    it("matches regardless of source case", () => {
+      const db = buildNewPackagesDatabase([
+        { source: "NPM", package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) },
+      ]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("matches entries with no source field", () => {
+      const db = buildNewPackagesDatabase([
+        { package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) },
+      ]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("respects a custom minimumPackageAgeHours threshold", () => {
+      minimumPackageAgeHours = 168; // 7 days
+
+      const db = buildNewPackagesDatabase([
+        { package_name: "foo", version: "1.0.0", released_on: hoursAgo(100) },
+      ]);
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+
+      minimumPackageAgeHours = 24; // reset
+    });
+  });
+});

packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.js

@@ -0,0 +1,17 @@
+import { ui } from "../environment/userInteraction.js";
+
+let hasWarnedAboutUnavailableNewPackagesDatabase = false;
+
+/** @param {Error} error */
+export function warnOnceAboutUnavailableDatabase(error) {
+  if (!hasWarnedAboutUnavailableNewPackagesDatabase) {
+    ui.writeWarning(
+      `Failed to load the new packages list used for direct package download request blocking. Continuing with metadata-based minimum age checks only. ${error.message}`
+    );
+    hasWarnedAboutUnavailableNewPackagesDatabase = true;
+  }
+}
+
+export function resetWarningState() {
+  hasWarnedAboutUnavailableNewPackagesDatabase = false;
+}

packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.spec.js

@@ -0,0 +1,63 @@
+import { describe, it, mock, beforeEach } from "node:test";
+import assert from "node:assert";
+
+let writeWarningCalls = [];
+
+mock.module("../environment/userInteraction.js", {
+  namedExports: {
+    ui: {
+      writeWarning: (msg) => writeWarningCalls.push(msg),
+    },
+  },
+});
+
+const { warnOnceAboutUnavailableDatabase, resetWarningState } = await import(
+  "./newPackagesDatabaseWarnings.js"
+);
+
+describe("newPackagesDatabaseWarnings", () => {
+  beforeEach(() => {
+    writeWarningCalls = [];
+    resetWarningState();
+  });
+
+  describe("warnOnceAboutUnavailableDatabase", () => {
+    it("emits a warning containing the error message", () => {
+      warnOnceAboutUnavailableDatabase(new Error("feed unavailable"));
+
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("feed unavailable"));
+    });
+
+    it("mentions fallback to metadata-based checks in the warning", () => {
+      warnOnceAboutUnavailableDatabase(new Error("timeout"));
+
+      assert.ok(
+        writeWarningCalls[0].includes(
+          "Continuing with metadata-based minimum age checks only"
+        )
+      );
+    });
+
+    it("only emits once even when called multiple times", () => {
+      warnOnceAboutUnavailableDatabase(new Error("first"));
+      warnOnceAboutUnavailableDatabase(new Error("second"));
+      warnOnceAboutUnavailableDatabase(new Error("third"));
+
+      assert.strictEqual(writeWarningCalls.length, 1);
+    });
+  });
+
+  describe("resetWarningState", () => {
+    it("allows the warning to fire again after reset", () => {
+      warnOnceAboutUnavailableDatabase(new Error("first"));
+      assert.strictEqual(writeWarningCalls.length, 1);
+
+      resetWarningState();
+      writeWarningCalls = [];
+
+      warnOnceAboutUnavailableDatabase(new Error("second"));
+      assert.strictEqual(writeWarningCalls.length, 1);
+    });
+  });
+});

packages/safe-chain/src/scanning/newPackagesListCache.js

@@ -8,40 +8,17 @@ import {
   getNewPackagesListVersionPath,
 } from "../config/configFile.js";
 import { ui } from "../environment/userInteraction.js";
-import {
+import { getEcoSystem, ECOSYSTEM_JS } from "../config/settings.js";
-  getMinimumPackageAgeHours,
+import { buildNewPackagesDatabase } from "./newPackagesDatabaseBuilder.js";
-  getEcoSystem,
+import { warnOnceAboutUnavailableDatabase } from "./newPackagesDatabaseWarnings.js";
-  ECOSYSTEM_JS,
-  ECOSYSTEM_PY,
-} from "../config/settings.js";
 
 /**
- * @typedef {Object} NewPackagesDatabase
+ * @typedef {import("./newPackagesDatabaseBuilder.js").NewPackagesDatabase} NewPackagesDatabase
- * @property {function(string, string): boolean} isNewlyReleasedPackage
  */
 
 // Shared per-process cache to avoid rebuilding the same feed-backed database on each request.
 /** @type {NewPackagesDatabase | null} */
 let cachedNewPackagesDatabase = null;
-let hasWarnedAboutUnavailableNewPackagesDatabase = false;
-
-/**
- * Returns the ecosystem identifier expected in upstream/core release feeds.
- * @returns {string}
- */
-function getCurrentFeedSource() {
-  const ecosystem = getEcoSystem();
-
-  if (ecosystem === ECOSYSTEM_JS) {
-    return "npm";
-  }
-
-  if (ecosystem === ECOSYSTEM_PY) {
-    return "pypi";
-  }
-
-  return ecosystem;
-}
 
 /**
  * @returns {Promise<NewPackagesDatabase>}
@@ -62,44 +39,12 @@ export async function openNewPackagesDatabase() {
   try {
     newPackagesList = await getNewPackagesList();
   } catch (/** @type {any} */ error) {
-    if (!hasWarnedAboutUnavailableNewPackagesDatabase) {
+    warnOnceAboutUnavailableDatabase(error);
-      ui.writeWarning(
-        `Failed to load the new packages list used for direct package download request blocking. Continuing with metadata-based minimum age checks only. ${error.message}`
-      );
-      hasWarnedAboutUnavailableNewPackagesDatabase = true;
-    }
-
     cachedNewPackagesDatabase = { isNewlyReleasedPackage: () => false };
     return cachedNewPackagesDatabase;
   }
 
-  /**
+  cachedNewPackagesDatabase = buildNewPackagesDatabase(newPackagesList);
-   * @param {string} name
-   * @param {string} version
-   * @returns {boolean}
-   */
-  function isNewlyReleasedPackage(name, version) {
-    const cutOff = new Date(
-      new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
-    );
-    const expectedSource = getCurrentFeedSource();
-
-    const entry = newPackagesList.find(
-      (pkg) =>
-        (!pkg.source || pkg.source.toLowerCase() === expectedSource) &&
-        pkg.package_name === name &&
-        pkg.version === version
-    );
-
-    if (!entry) {
-      return false;
-    }
-
-    const releasedOn = new Date(entry.released_on * 1000);
-    return releasedOn > cutOff;
-  }
-
-  cachedNewPackagesDatabase = { isNewlyReleasedPackage };
   return cachedNewPackagesDatabase;
 }
 

packages/safe-chain/src/scanning/newPackagesListCache.spec.js

@@ -0,0 +1,177 @@
+import { describe, it, mock, beforeEach } from "node:test";
+import assert from "node:assert";
+import fs from "fs";
+import path from "path";
+import os from "os";
+
+let writeWarningCalls = [];
+let ecosystem = "js";
+let testHomeDir = "";
+
+mock.module("../environment/userInteraction.js", {
+  namedExports: {
+    ui: {
+      writeWarning: (msg) => writeWarningCalls.push(msg),
+    },
+  },
+});
+
+mock.module("../config/settings.js", {
+  namedExports: {
+    getEcoSystem: () => ecosystem,
+    getMinimumPackageAgeHours: () => 24,
+    ECOSYSTEM_JS: "js",
+    ECOSYSTEM_PY: "py",
+  },
+});
+
+const { readNewPackagesListFromLocalCache, writeNewPackagesListToLocalCache } =
+  await import("./newPackagesListCache.js");
+
+describe("newPackagesListCache", () => {
+  beforeEach(() => {
+    writeWarningCalls = [];
+    ecosystem = "js";
+    testHomeDir = path.join(
+      os.tmpdir(),
+      `safe-chain-list-cache-${process.pid}-${Date.now()}`
+    );
+    fs.rmSync(testHomeDir, { recursive: true, force: true });
+    fs.mkdirSync(testHomeDir, { recursive: true });
+    process.env.HOME = testHomeDir;
+  });
+
+  describe("readNewPackagesListFromLocalCache", () => {
+    it("returns null for both fields when no cache file exists", () => {
+      const result = readNewPackagesListFromLocalCache();
+
+      assert.deepStrictEqual(result, { newPackagesList: null, version: null });
+    });
+
+    it("returns the list and version when both files exist", () => {
+      const list = [{ package_name: "foo", version: "1.0.0" }];
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_js.json"),
+        JSON.stringify(list)
+      );
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_version_js.txt"),
+        "etag-42"
+      );
+
+      const result = readNewPackagesListFromLocalCache();
+
+      assert.deepStrictEqual(result.newPackagesList, list);
+      assert.strictEqual(result.version, "etag-42");
+    });
+
+    it("returns null version when version file is missing", () => {
+      const list = [{ package_name: "foo", version: "1.0.0" }];
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_js.json"),
+        JSON.stringify(list)
+      );
+
+      const result = readNewPackagesListFromLocalCache();
+
+      assert.deepStrictEqual(result.newPackagesList, list);
+      assert.strictEqual(result.version, null);
+    });
+
+    it("trims whitespace from the version string", () => {
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_js.json"),
+        JSON.stringify([])
+      );
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_version_js.txt"),
+        "  etag-trimmed  \n"
+      );
+
+      const { version } = readNewPackagesListFromLocalCache();
+
+      assert.strictEqual(version, "etag-trimmed");
+    });
+
+    it("uses the ecosystem name in the file path", () => {
+      ecosystem = "py";
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_py.json"),
+        JSON.stringify([{ package_name: "requests", version: "2.0.0" }])
+      );
+
+      const result = readNewPackagesListFromLocalCache();
+
+      assert.ok(result.newPackagesList !== null);
+    });
+
+    it("warns and returns nulls when the list file contains invalid JSON", () => {
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(safeChainDir, "newPackagesList_js.json"),
+        "not-valid-json"
+      );
+
+      const result = readNewPackagesListFromLocalCache();
+
+      assert.deepStrictEqual(result, { newPackagesList: null, version: null });
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("local cache"));
+    });
+  });
+
+  describe("writeNewPackagesListToLocalCache", () => {
+    it("writes the list and version to disk", () => {
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+
+      const list = [{ package_name: "foo", version: "1.0.0" }];
+      writeNewPackagesListToLocalCache(list, "etag-99");
+
+      const writtenList = JSON.parse(
+        fs.readFileSync(path.join(safeChainDir, "newPackagesList_js.json"), "utf8")
+      );
+      const writtenVersion = fs.readFileSync(
+        path.join(safeChainDir, "newPackagesList_version_js.txt"),
+        "utf8"
+      );
+
+      assert.deepStrictEqual(writtenList, list);
+      assert.strictEqual(writtenVersion, "etag-99");
+    });
+
+    it("converts a numeric version to a string", () => {
+      const safeChainDir = path.join(testHomeDir, ".safe-chain");
+      fs.mkdirSync(safeChainDir, { recursive: true });
+
+      writeNewPackagesListToLocalCache([], 42);
+
+      const written = fs.readFileSync(
+        path.join(safeChainDir, "newPackagesList_version_js.txt"),
+        "utf8"
+      );
+      assert.strictEqual(written, "42");
+    });
+
+    it("warns when writing fails", () => {
+      // Place a regular file at the .safe-chain path so getSafeChainDirectory
+      // returns it as-is (existsSync is true) but writing a child path fails.
+      const safeChainPath = path.join(testHomeDir, ".safe-chain");
+      fs.writeFileSync(safeChainPath, "not-a-directory");
+
+      writeNewPackagesListToLocalCache([], "etag-fail");
+
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("local cache"));
+    });
+  });
+});