Fetch new package list

packages/safe-chain/src/api/aikido.js

@@ -11,6 +11,13 @@ const malwareDatabaseUrls = {
   [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/malware_pypi.json",
 };
 
+// TODO: replace with the real CDN URL once core publishes the S3 endpoint
+const newPackagesListUrls = {
+  [ECOSYSTEM_JS]: "https://new-packages.aikido.dev/js_packages.json",
+};
+
+const DEFAULT_FETCH_RETRY_ATTEMPTS = 4;
+
 /**
  * @typedef {Object} MalwarePackage
  * @property {string} package_name
@@ -18,12 +25,19 @@ const malwareDatabaseUrls = {
  * @property {string} reason
  */
 
+/**
+ * @typedef {Object} NewPackageEntry
+ * @property {string} source
+ * @property {string} name
+ * @property {string} version
+ * @property {number} released_on  - Unix timestamp (seconds)
+ * @property {number} scraped_on   - Unix timestamp (seconds)
+ */
+
 /**
  * @returns {Promise<{malwareDatabase: MalwarePackage[], version: string | undefined}>}
  */
 export async function fetchMalwareDatabase() {
-  const numberOfAttempts = 4;
-
   return retry(async () => {
     const ecosystem = getEcoSystem();
     const malwareDatabaseUrl =
@@ -46,15 +60,13 @@ export async function fetchMalwareDatabase() {
     } catch (/** @type {any} */ error) {
       throw new Error(`Error parsing malware database: ${error.message}`);
     }
-  }, numberOfAttempts);
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
 }
 
 /**
  * @returns {Promise<string | undefined>}
  */
 export async function fetchMalwareDatabaseVersion() {
-  const numberOfAttempts = 4;
-
   return retry(async () => {
     const ecosystem = getEcoSystem();
     const malwareDatabaseUrl =
@@ -71,7 +83,63 @@ export async function fetchMalwareDatabaseVersion() {
       );
     }
     return response.headers.get("etag") || undefined;
-  }, numberOfAttempts);
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
+}
+
+/**
+ * @returns {Promise<{newPackagesList: NewPackageEntry[], version: string | undefined}>}
+ */
+export async function fetchNewPackagesList() {
+  return retry(async () => {
+    const ecosystem = getEcoSystem();
+    const url =
+      newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)];
+
+    if (!url) {
+      return { newPackagesList: [], version: undefined };
+    }
+
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(
+        `Error fetching ${ecosystem} new packages list: ${response.statusText}`
+      );
+    }
+
+    try {
+      const newPackagesList = await response.json();
+      return {
+        newPackagesList,
+        version: response.headers.get("etag") || undefined,
+      };
+    } catch (/** @type {any} */ error) {
+      throw new Error(`Error parsing new packages list: ${error.message}`);
+    }
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
+}
+
+/**
+ * @returns {Promise<string | undefined>}
+ */
+export async function fetchNewPackagesListVersion() {
+  return retry(async () => {
+    const ecosystem = getEcoSystem();
+    const url =
+      newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)];
+
+    if (!url) {
+      return undefined;
+    }
+
+    const response = await fetch(url, { method: "HEAD" });
+    if (!response.ok) {
+      throw new Error(
+        `Error fetching ${ecosystem} new packages list version: ${response.statusText}`
+      );
+    }
+
+    return response.headers.get("etag") || undefined;
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
 }
 
 /**
@@ -91,7 +159,7 @@ async function retry(func, attempts) {
       return await func();
     } catch (error) {
       ui.writeVerbose(
-        "An error occurred while trying to download the Aikido Malware database",
+        "An error occurred while trying to download Aikido data",
         error
       );
       lastError = error;

packages/safe-chain/src/api/aikido.spec.js

@@ -3,6 +3,7 @@ import assert from "node:assert";
 
 describe("aikido API", async () => {
   const mockFetch = mock.fn();
+  let ecosystem = "js";
 
   mock.module("make-fetch-happen", {
     defaultExport: mockFetch,
@@ -18,17 +19,22 @@ describe("aikido API", async () => {
 
   mock.module("../config/settings.js", {
     namedExports: {
-      getEcoSystem: () => "js",
+      getEcoSystem: () => ecosystem,
       ECOSYSTEM_JS: "js",
       ECOSYSTEM_PY: "py",
     },
   });
 
-  const { fetchMalwareDatabase, fetchMalwareDatabaseVersion } =
-    await import("./aikido.js");
+  const {
+    fetchMalwareDatabase,
+    fetchMalwareDatabaseVersion,
+    fetchNewPackagesList,
+    fetchNewPackagesListVersion,
+  } = await import("./aikido.js");
 
   beforeEach(() => {
     mockFetch.mock.resetCalls();
+    ecosystem = "js";
   });
 
   describe("fetchMalwareDatabase", () => {
@@ -130,4 +136,77 @@ describe("aikido API", async () => {
       assert.strictEqual(result, '"final-etag"');
     });
   });
+
+  describe("fetchNewPackagesList", () => {
+    it("should succeed immediately when fetch succeeds on first try", async () => {
+      const releases = [
+        {
+          source: "NPM",
+          name: "fresh-pkg",
+          version: "1.0.0",
+          released_on: 123,
+          scraped_on: 456,
+        },
+      ];
+      mockFetch.mock.mockImplementationOnce(() => ({
+        ok: true,
+        json: async () => releases,
+        headers: { get: () => '"etag-new-packages"' },
+      }));
+
+      const result = await fetchNewPackagesList();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 1);
+      assert.deepStrictEqual(result.newPackagesList, releases);
+      assert.strictEqual(result.version, '"etag-new-packages"');
+    });
+
+    it("should throw error after exhausting all retries", async () => {
+      mockFetch.mock.mockImplementation(() => {
+        throw new Error("Network error");
+      });
+
+      await assert.rejects(() => fetchNewPackagesList(), {
+        message: "Network error",
+      });
+
+      assert.strictEqual(mockFetch.mock.calls.length, 4);
+    });
+
+    it("should return an empty list without fetching for unsupported ecosystems", async () => {
+      ecosystem = "py";
+
+      const result = await fetchNewPackagesList();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 0);
+      assert.deepStrictEqual(result.newPackagesList, []);
+      assert.strictEqual(result.version, undefined);
+    });
+  });
+
+  describe("fetchNewPackagesListVersion", () => {
+    it("should succeed immediately when fetch succeeds on first try", async () => {
+      mockFetch.mock.mockImplementationOnce(() => ({
+        ok: true,
+        headers: { get: () => '"new-packages-etag"' },
+      }));
+
+      const result = await fetchNewPackagesListVersion();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 1);
+      assert.strictEqual(result, '"new-packages-etag"');
+    });
+
+    it("should throw error after exhausting all retries", async () => {
+      mockFetch.mock.mockImplementation(() => {
+        throw new Error("Connection refused");
+      });
+
+      await assert.rejects(() => fetchNewPackagesListVersion(), {
+        message: "Connection refused",
+      });
+
+      assert.strictEqual(mockFetch.mock.calls.length, 4);
+    });
+  });
 });

packages/safe-chain/src/config/configFile.js

@@ -203,6 +203,70 @@ export function readDatabaseFromLocalCache() {
   }
 }
 
+/**
+ * @param {import("../api/aikido.js").NewPackageEntry[]} data
+ * @param {string | number} version
+ *
+ * @returns {void}
+ */
+export function writeNewPackagesListToLocalCache(data, version) {
+  try {
+    const listPath = getNewPackagesListPath();
+    const versionPath = getNewPackagesListVersionPath();
+
+    fs.writeFileSync(listPath, JSON.stringify(data));
+    fs.writeFileSync(versionPath, version.toString());
+  } catch {
+    ui.writeWarning(
+      "Failed to write new packages list to local cache, next time the list will be fetched from the server again."
+    );
+  }
+}
+
+/**
+ * @returns {{newPackagesList: import("../api/aikido.js").NewPackageEntry[] | null, version: string | null}}
+ */
+export function readNewPackagesListFromLocalCache() {
+  try {
+    const listPath = getNewPackagesListPath();
+    if (!fs.existsSync(listPath)) {
+      return { newPackagesList: null, version: null };
+    }
+
+    const data = fs.readFileSync(listPath, "utf8");
+    const newPackagesList = JSON.parse(data);
+    const versionPath = getNewPackagesListVersionPath();
+    let version = null;
+    if (fs.existsSync(versionPath)) {
+      version = fs.readFileSync(versionPath, "utf8").trim();
+    }
+    return { newPackagesList, version };
+  } catch {
+    ui.writeWarning(
+      "Failed to read new packages list from local cache. Continuing without local cache."
+    );
+    return { newPackagesList: null, version: null };
+  }
+}
+
+/**
+ * @returns {string}
+ */
+function getNewPackagesListPath() {
+  const safeChainDir = getSafeChainDirectory();
+  const ecosystem = getEcoSystem();
+  return path.join(safeChainDir, `newPackagesList_${ecosystem}.json`);
+}
+
+/**
+ * @returns {string}
+ */
+function getNewPackagesListVersionPath() {
+  const safeChainDir = getSafeChainDirectory();
+  const ecosystem = getEcoSystem();
+  return path.join(safeChainDir, `newPackagesList_version_${ecosystem}.txt`);
+}
+
 /**
  * @returns {SafeChainConfig}
  */

packages/safe-chain/src/scanning/newPackagesDatabase.js

@@ -0,0 +1,112 @@
+import {
+  fetchNewPackagesList,
+  fetchNewPackagesListVersion,
+} from "../api/aikido.js";
+import {
+  readNewPackagesListFromLocalCache,
+  writeNewPackagesListToLocalCache,
+} from "../config/configFile.js";
+import { ui } from "../environment/userInteraction.js";
+import {
+  getMinimumPackageAgeHours,
+  getEcoSystem,
+  ECOSYSTEM_JS,
+} from "../config/settings.js";
+
+/**
+ * @typedef {Object} NewPackagesDatabase
+ * @property {function(string, string): boolean} isNewlyReleasedPackage
+ */
+
+/** @type {NewPackagesDatabase | null} */
+let cachedNewPackagesDatabase = null;
+
+/**
+ * Returns the source identifier used in the feed for the current ecosystem.
+ * @returns {string}
+ */
+function getCurrentFeedSource() {
+  return getEcoSystem();
+}
+
+/**
+ * @returns {Promise<NewPackagesDatabase>}
+ */
+export async function openNewPackagesDatabase() {
+  if (cachedNewPackagesDatabase) {
+    return cachedNewPackagesDatabase;
+  }
+
+  if (getEcoSystem() !== ECOSYSTEM_JS) {
+    cachedNewPackagesDatabase = { isNewlyReleasedPackage: () => false };
+    return cachedNewPackagesDatabase;
+  }
+
+  const newPackagesList = await getNewPackagesList();
+
+  /**
+   * @param {string} name
+   * @param {string} version
+   * @returns {boolean}
+   */
+  function isNewlyReleasedPackage(name, version) {
+    const cutOff = new Date(
+      new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
+    );
+    const expectedSource = getCurrentFeedSource();
+
+    const entry = newPackagesList.find(
+      (pkg) =>
+        pkg.source?.toLowerCase() === expectedSource &&
+        pkg.name === name &&
+        pkg.version === version
+    );
+
+    if (!entry) {
+      return false;
+    }
+
+    const releasedOn = new Date(entry.released_on * 1000);
+    return releasedOn > cutOff;
+  }
+
+  cachedNewPackagesDatabase = { isNewlyReleasedPackage };
+  return cachedNewPackagesDatabase;
+}
+
+/**
+ * @returns {Promise<import("../api/aikido.js").NewPackageEntry[]>}
+ */
+async function getNewPackagesList() {
+  const { newPackagesList: cachedList, version: cachedVersion } =
+    readNewPackagesListFromLocalCache();
+
+  try {
+    if (cachedList) {
+      const currentVersion = await fetchNewPackagesListVersion();
+      if (cachedVersion === currentVersion) {
+        return cachedList;
+      }
+    }
+
+    const { newPackagesList, version } = await fetchNewPackagesList();
+
+    if (version) {
+      writeNewPackagesListToLocalCache(newPackagesList, version);
+      return newPackagesList;
+    } else {
+      ui.writeWarning(
+        "The new packages list was downloaded, but could not be cached due to a missing version."
+      );
+      return newPackagesList;
+    }
+  } catch (/** @type {any} */ error) {
+    if (cachedList) {
+      ui.writeWarning(
+        "Failed to fetch the latest new packages list. Using cached version."
+      );
+      return cachedList;
+    }
+    throw error;
+  }
+}

packages/safe-chain/src/scanning/newPackagesDatabase.spec.js

@@ -0,0 +1,230 @@
+import { describe, it, mock, beforeEach } from "node:test";
+import assert from "node:assert";
+
+// --- shared mutable state for mocks ---
+let cachedList = null;
+let cachedVersion = null;
+let fetchedList = [];
+let fetchedVersion = "etag-1";
+let fetchVersionResult = "etag-1";
+let minimumPackageAgeHours = 24;
+let ecosystem = "js";
+let writeWarningCalls = [];
+let fetchListError = null;
+let fetchVersionError = null;
+let importCounter = 0;
+
+mock.module("../api/aikido.js", {
+  namedExports: {
+    fetchNewPackagesList: async () => {
+      if (fetchListError) {
+        throw fetchListError;
+      }
+
+      return {
+        newPackagesList: fetchedList,
+        version: fetchedVersion,
+      };
+    },
+    fetchNewPackagesListVersion: async () => {
+      if (fetchVersionError) {
+        throw fetchVersionError;
+      }
+
+      return fetchVersionResult;
+    },
+  },
+});
+
+mock.module("../config/configFile.js", {
+  namedExports: {
+    readNewPackagesListFromLocalCache: () => ({
+      newPackagesList: cachedList,
+      version: cachedVersion,
+    }),
+    writeNewPackagesListToLocalCache: () => {},
+  },
+});
+
+mock.module("../environment/userInteraction.js", {
+  namedExports: {
+    ui: {
+      writeWarning: (msg) => writeWarningCalls.push(msg),
+      writeVerbose: () => {},
+    },
+  },
+});
+
+mock.module("../config/settings.js", {
+  namedExports: {
+    getMinimumPackageAgeHours: () => minimumPackageAgeHours,
+    getEcoSystem: () => ecosystem,
+    ECOSYSTEM_JS: "js",
+    ECOSYSTEM_PY: "py",
+  },
+});
+
+describe("newPackagesDatabase", async () => {
+  beforeEach(() => {
+    cachedList = null;
+    cachedVersion = null;
+    fetchedList = [];
+    fetchedVersion = "etag-1";
+    fetchVersionResult = "etag-1";
+    minimumPackageAgeHours = 24;
+    ecosystem = "js";
+    writeWarningCalls = [];
+    fetchListError = null;
+    fetchVersionError = null;
+  });
+
+  async function openNewPackagesDatabase() {
+    const module = await import(
+      `./newPackagesDatabase.js?test_case=${importCounter++}`
+    );
+    return module.openNewPackagesDatabase();
+  }
+
+  function hoursAgo(hours) {
+    return Math.floor((Date.now() - hours * 3600 * 1000) / 1000);
+  }
+
+  describe("isNewlyReleasedPackage", () => {
+    it("returns true for a package released within the age threshold", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("returns false for a package released outside the age threshold", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(48), scraped_on: hoursAgo(48) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+
+    it("returns false for a package not in the list", async () => {
+      fetchedList = [];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("not-there", "1.0.0"), false);
+    });
+
+    it("returns false for a known package but different version", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+
+    it("ignores entries from a different source in a mixed feed", async () => {
+      fetchedList = [
+        {
+          source: "npm",
+          name: "foo",
+          version: "1.0.0",
+          released_on: hoursAgo(1),
+          scraped_on: hoursAgo(1),
+        },
+        {
+          source: "js",
+          name: "bar",
+          version: "1.0.0",
+          released_on: hoursAgo(1),
+          scraped_on: hoursAgo(1),
+        },
+      ];
+
+      const db = await openNewPackagesDatabase();
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+      assert.strictEqual(db.isNewlyReleasedPackage("bar", "1.0.0"), true);
+    });
+
+    it("respects a custom minimumPackageAgeHours threshold", async () => {
+      minimumPackageAgeHours = 168; // 7 days
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(100), scraped_on: hoursAgo(100) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("returns false for all packages when ecosystem is not JS", async () => {
+      ecosystem = "py";
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+  });
+
+  describe("caching behaviour", () => {
+    it("uses local cache when etag matches", async () => {
+      cachedList = [
+        { source: "js", name: "cached-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+      cachedVersion = "etag-1";
+      fetchVersionResult = "etag-1";
+      // fetchedList is empty — if we used the remote list, the lookup would return false
+      fetchedList = [];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("cached-pkg", "1.0.0"), true);
+    });
+
+    it("fetches fresh list when etag does not match", async () => {
+      cachedList = [
+        { source: "js", name: "stale-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+      cachedVersion = "etag-old";
+      fetchVersionResult = "etag-new";
+      fetchedList = [
+        { source: "js", name: "fresh-pkg", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("stale-pkg", "1.0.0"), false);
+      assert.strictEqual(db.isNewlyReleasedPackage("fresh-pkg", "2.0.0"), true);
+    });
+
+    it("falls back to local cache when fetch fails", async () => {
+      cachedList = [
+        {
+          source: "js",
+          name: "cached-pkg",
+          version: "1.0.0",
+          released_on: hoursAgo(1),
+          scraped_on: hoursAgo(1),
+        },
+      ];
+      cachedVersion = "etag-old";
+      fetchVersionResult = "etag-new";
+      fetchListError = new Error("Network error");
+
+      const db = await openNewPackagesDatabase();
+
+      assert.strictEqual(db.isNewlyReleasedPackage("cached-pkg", "1.0.0"), true);
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("Using cached version"));
+    });
+
+    it("emits a warning when list has no version (cannot be cached)", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+      fetchedVersion = undefined;
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("could not be cached"));
+    });
+  });
+});

packages/safe-chain/src/ultimate/ultimateTroubleshooting.js

@@ -42,7 +42,7 @@ export async function troubleshootingExport() {
       resolve(zipFileName);
     });
 
-    archive.on('error', (err) => {
+    archive.on('error', (/** @type {Error} */ err) => {
       ui.writeError(`Failed to zip logs: ${err.message}`);
       reject(err);
     });