Fetch new package list

packages/safe-chain/src/scanning/newPackagesDatabase.js

@@ -0,0 +1,112 @@
+import {
+  fetchNewPackagesList,
+  fetchNewPackagesListVersion,
+} from "../api/aikido.js";
+import {
+  readNewPackagesListFromLocalCache,
+  writeNewPackagesListToLocalCache,
+} from "../config/configFile.js";
+import { ui } from "../environment/userInteraction.js";
+import {
+  getMinimumPackageAgeHours,
+  getEcoSystem,
+  ECOSYSTEM_JS,
+} from "../config/settings.js";
+
+/**
+ * @typedef {Object} NewPackagesDatabase
+ * @property {function(string, string): boolean} isNewlyReleasedPackage
+ */
+
+/** @type {NewPackagesDatabase | null} */
+let cachedNewPackagesDatabase = null;
+
+/**
+ * Returns the source identifier used in the feed for the current ecosystem.
+ * @returns {string}
+ */
+function getCurrentFeedSource() {
+  return getEcoSystem();
+}
+
+/**
+ * @returns {Promise<NewPackagesDatabase>}
+ */
+export async function openNewPackagesDatabase() {
+  if (cachedNewPackagesDatabase) {
+    return cachedNewPackagesDatabase;
+  }
+
+  if (getEcoSystem() !== ECOSYSTEM_JS) {
+    cachedNewPackagesDatabase = { isNewlyReleasedPackage: () => false };
+    return cachedNewPackagesDatabase;
+  }
+
+  const newPackagesList = await getNewPackagesList();
+
+  /**
+   * @param {string} name
+   * @param {string} version
+   * @returns {boolean}
+   */
+  function isNewlyReleasedPackage(name, version) {
+    const cutOff = new Date(
+      new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
+    );
+    const expectedSource = getCurrentFeedSource();
+
+    const entry = newPackagesList.find(
+      (pkg) =>
+        pkg.source?.toLowerCase() === expectedSource &&
+        pkg.name === name &&
+        pkg.version === version
+    );
+
+    if (!entry) {
+      return false;
+    }
+
+    const releasedOn = new Date(entry.released_on * 1000);
+    return releasedOn > cutOff;
+  }
+
+  cachedNewPackagesDatabase = { isNewlyReleasedPackage };
+  return cachedNewPackagesDatabase;
+}
+
+/**
+ * @returns {Promise<import("../api/aikido.js").NewPackageEntry[]>}
+ */
+async function getNewPackagesList() {
+  const { newPackagesList: cachedList, version: cachedVersion } =
+    readNewPackagesListFromLocalCache();
+
+  try {
+    if (cachedList) {
+      const currentVersion = await fetchNewPackagesListVersion();
+      if (cachedVersion === currentVersion) {
+        return cachedList;
+      }
+    }
+
+    const { newPackagesList, version } = await fetchNewPackagesList();
+
+    if (version) {
+      writeNewPackagesListToLocalCache(newPackagesList, version);
+      return newPackagesList;
+    } else {
+      ui.writeWarning(
+        "The new packages list was downloaded, but could not be cached due to a missing version."
+      );
+      return newPackagesList;
+    }
+  } catch (/** @type {any} */ error) {
+    if (cachedList) {
+      ui.writeWarning(
+        "Failed to fetch the latest new packages list. Using cached version."
+      );
+      return cachedList;
+    }
+    throw error;
+  }
+}

packages/safe-chain/src/scanning/newPackagesDatabase.spec.js

@@ -0,0 +1,230 @@
+import { describe, it, mock, beforeEach } from "node:test";
+import assert from "node:assert";
+
+// --- shared mutable state for mocks ---
+let cachedList = null;
+let cachedVersion = null;
+let fetchedList = [];
+let fetchedVersion = "etag-1";
+let fetchVersionResult = "etag-1";
+let minimumPackageAgeHours = 24;
+let ecosystem = "js";
+let writeWarningCalls = [];
+let fetchListError = null;
+let fetchVersionError = null;
+let importCounter = 0;
+
+mock.module("../api/aikido.js", {
+  namedExports: {
+    fetchNewPackagesList: async () => {
+      if (fetchListError) {
+        throw fetchListError;
+      }
+
+      return {
+        newPackagesList: fetchedList,
+        version: fetchedVersion,
+      };
+    },
+    fetchNewPackagesListVersion: async () => {
+      if (fetchVersionError) {
+        throw fetchVersionError;
+      }
+
+      return fetchVersionResult;
+    },
+  },
+});
+
+mock.module("../config/configFile.js", {
+  namedExports: {
+    readNewPackagesListFromLocalCache: () => ({
+      newPackagesList: cachedList,
+      version: cachedVersion,
+    }),
+    writeNewPackagesListToLocalCache: () => {},
+  },
+});
+
+mock.module("../environment/userInteraction.js", {
+  namedExports: {
+    ui: {
+      writeWarning: (msg) => writeWarningCalls.push(msg),
+      writeVerbose: () => {},
+    },
+  },
+});
+
+mock.module("../config/settings.js", {
+  namedExports: {
+    getMinimumPackageAgeHours: () => minimumPackageAgeHours,
+    getEcoSystem: () => ecosystem,
+    ECOSYSTEM_JS: "js",
+    ECOSYSTEM_PY: "py",
+  },
+});
+
+describe("newPackagesDatabase", async () => {
+  beforeEach(() => {
+    cachedList = null;
+    cachedVersion = null;
+    fetchedList = [];
+    fetchedVersion = "etag-1";
+    fetchVersionResult = "etag-1";
+    minimumPackageAgeHours = 24;
+    ecosystem = "js";
+    writeWarningCalls = [];
+    fetchListError = null;
+    fetchVersionError = null;
+  });
+
+  async function openNewPackagesDatabase() {
+    const module = await import(
+      `./newPackagesDatabase.js?test_case=${importCounter++}`
+    );
+    return module.openNewPackagesDatabase();
+  }
+
+  function hoursAgo(hours) {
+    return Math.floor((Date.now() - hours * 3600 * 1000) / 1000);
+  }
+
+  describe("isNewlyReleasedPackage", () => {
+    it("returns true for a package released within the age threshold", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("returns false for a package released outside the age threshold", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(48), scraped_on: hoursAgo(48) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+
+    it("returns false for a package not in the list", async () => {
+      fetchedList = [];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("not-there", "1.0.0"), false);
+    });
+
+    it("returns false for a known package but different version", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+
+    it("ignores entries from a different source in a mixed feed", async () => {
+      fetchedList = [
+        {
+          source: "npm",
+          name: "foo",
+          version: "1.0.0",
+          released_on: hoursAgo(1),
+          scraped_on: hoursAgo(1),
+        },
+        {
+          source: "js",
+          name: "bar",
+          version: "1.0.0",
+          released_on: hoursAgo(1),
+          scraped_on: hoursAgo(1),
+        },
+      ];
+
+      const db = await openNewPackagesDatabase();
+
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+      assert.strictEqual(db.isNewlyReleasedPackage("bar", "1.0.0"), true);
+    });
+
+    it("respects a custom minimumPackageAgeHours threshold", async () => {
+      minimumPackageAgeHours = 168; // 7 days
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(100), scraped_on: hoursAgo(100) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+    });
+
+    it("returns false for all packages when ecosystem is not JS", async () => {
+      ecosystem = "py";
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
+    });
+  });
+
+  describe("caching behaviour", () => {
+    it("uses local cache when etag matches", async () => {
+      cachedList = [
+        { source: "js", name: "cached-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+      cachedVersion = "etag-1";
+      fetchVersionResult = "etag-1";
+      // fetchedList is empty — if we used the remote list, the lookup would return false
+      fetchedList = [];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("cached-pkg", "1.0.0"), true);
+    });
+
+    it("fetches fresh list when etag does not match", async () => {
+      cachedList = [
+        { source: "js", name: "stale-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+      cachedVersion = "etag-old";
+      fetchVersionResult = "etag-new";
+      fetchedList = [
+        { source: "js", name: "fresh-pkg", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("stale-pkg", "1.0.0"), false);
+      assert.strictEqual(db.isNewlyReleasedPackage("fresh-pkg", "2.0.0"), true);
+    });
+
+    it("falls back to local cache when fetch fails", async () => {
+      cachedList = [
+        {
+          source: "js",
+          name: "cached-pkg",
+          version: "1.0.0",
+          released_on: hoursAgo(1),
+          scraped_on: hoursAgo(1),
+        },
+      ];
+      cachedVersion = "etag-old";
+      fetchVersionResult = "etag-new";
+      fetchListError = new Error("Network error");
+
+      const db = await openNewPackagesDatabase();
+
+      assert.strictEqual(db.isNewlyReleasedPackage("cached-pkg", "1.0.0"), true);
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("Using cached version"));
+    });
+
+    it("emits a warning when list has no version (cannot be cached)", async () => {
+      fetchedList = [
+        { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) },
+      ];
+      fetchedVersion = undefined;
+
+      const db = await openNewPackagesDatabase();
+      assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
+      assert.strictEqual(writeWarningCalls.length, 1);
+      assert.ok(writeWarningCalls[0].includes("could not be cached"));
+    });
+  });
+});