Fetch new package list

packages/safe-chain/src/api/aikido.js

@@ -11,6 +11,13 @@ const malwareDatabaseUrls = {
   [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/malware_pypi.json",
 };
 
+// TODO: replace with the real CDN URL once core publishes the S3 endpoint
+const newPackagesListUrls = {
+  [ECOSYSTEM_JS]: "https://new-packages.aikido.dev/js_packages.json",
+};
+
+const DEFAULT_FETCH_RETRY_ATTEMPTS = 4;
+
 /**
  * @typedef {Object} MalwarePackage
  * @property {string} package_name
@@ -18,12 +25,19 @@ const malwareDatabaseUrls = {
  * @property {string} reason
  */
 
+/**
+ * @typedef {Object} NewPackageEntry
+ * @property {string} source
+ * @property {string} name
+ * @property {string} version
+ * @property {number} released_on  - Unix timestamp (seconds)
+ * @property {number} scraped_on   - Unix timestamp (seconds)
+ */
+
 /**
  * @returns {Promise<{malwareDatabase: MalwarePackage[], version: string | undefined}>}
  */
 export async function fetchMalwareDatabase() {
-  const numberOfAttempts = 4;
-
   return retry(async () => {
     const ecosystem = getEcoSystem();
     const malwareDatabaseUrl =
@@ -46,15 +60,13 @@ export async function fetchMalwareDatabase() {
     } catch (/** @type {any} */ error) {
       throw new Error(`Error parsing malware database: ${error.message}`);
     }
-  }, numberOfAttempts);
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
 }
 
 /**
  * @returns {Promise<string | undefined>}
  */
 export async function fetchMalwareDatabaseVersion() {
-  const numberOfAttempts = 4;
-
   return retry(async () => {
     const ecosystem = getEcoSystem();
     const malwareDatabaseUrl =
@@ -71,7 +83,63 @@ export async function fetchMalwareDatabaseVersion() {
       );
     }
     return response.headers.get("etag") || undefined;
-  }, numberOfAttempts);
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
+}
+
+/**
+ * @returns {Promise<{newPackagesList: NewPackageEntry[], version: string | undefined}>}
+ */
+export async function fetchNewPackagesList() {
+  return retry(async () => {
+    const ecosystem = getEcoSystem();
+    const url =
+      newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)];
+
+    if (!url) {
+      return { newPackagesList: [], version: undefined };
+    }
+
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(
+        `Error fetching ${ecosystem} new packages list: ${response.statusText}`
+      );
+    }
+
+    try {
+      const newPackagesList = await response.json();
+      return {
+        newPackagesList,
+        version: response.headers.get("etag") || undefined,
+      };
+    } catch (/** @type {any} */ error) {
+      throw new Error(`Error parsing new packages list: ${error.message}`);
+    }
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
+}
+
+/**
+ * @returns {Promise<string | undefined>}
+ */
+export async function fetchNewPackagesListVersion() {
+  return retry(async () => {
+    const ecosystem = getEcoSystem();
+    const url =
+      newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)];
+
+    if (!url) {
+      return undefined;
+    }
+
+    const response = await fetch(url, { method: "HEAD" });
+    if (!response.ok) {
+      throw new Error(
+        `Error fetching ${ecosystem} new packages list version: ${response.statusText}`
+      );
+    }
+
+    return response.headers.get("etag") || undefined;
+  }, DEFAULT_FETCH_RETRY_ATTEMPTS);
 }
 
 /**
@@ -91,7 +159,7 @@ async function retry(func, attempts) {
       return await func();
     } catch (error) {
       ui.writeVerbose(
-        "An error occurred while trying to download the Aikido Malware database",
+        "An error occurred while trying to download Aikido data",
         error
       );
       lastError = error;

packages/safe-chain/src/api/aikido.spec.js

@@ -3,6 +3,7 @@ import assert from "node:assert";
 
 describe("aikido API", async () => {
   const mockFetch = mock.fn();
+  let ecosystem = "js";
 
   mock.module("make-fetch-happen", {
     defaultExport: mockFetch,
@@ -18,17 +19,22 @@ describe("aikido API", async () => {
 
   mock.module("../config/settings.js", {
     namedExports: {
-      getEcoSystem: () => "js",
+      getEcoSystem: () => ecosystem,
       ECOSYSTEM_JS: "js",
       ECOSYSTEM_PY: "py",
     },
   });
 
-  const { fetchMalwareDatabase, fetchMalwareDatabaseVersion } =
-    await import("./aikido.js");
+  const {
+    fetchMalwareDatabase,
+    fetchMalwareDatabaseVersion,
+    fetchNewPackagesList,
+    fetchNewPackagesListVersion,
+  } = await import("./aikido.js");
 
   beforeEach(() => {
     mockFetch.mock.resetCalls();
+    ecosystem = "js";
   });
 
   describe("fetchMalwareDatabase", () => {
@@ -130,4 +136,77 @@ describe("aikido API", async () => {
       assert.strictEqual(result, '"final-etag"');
     });
   });
+
+  describe("fetchNewPackagesList", () => {
+    it("should succeed immediately when fetch succeeds on first try", async () => {
+      const releases = [
+        {
+          source: "NPM",
+          name: "fresh-pkg",
+          version: "1.0.0",
+          released_on: 123,
+          scraped_on: 456,
+        },
+      ];
+      mockFetch.mock.mockImplementationOnce(() => ({
+        ok: true,
+        json: async () => releases,
+        headers: { get: () => '"etag-new-packages"' },
+      }));
+
+      const result = await fetchNewPackagesList();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 1);
+      assert.deepStrictEqual(result.newPackagesList, releases);
+      assert.strictEqual(result.version, '"etag-new-packages"');
+    });
+
+    it("should throw error after exhausting all retries", async () => {
+      mockFetch.mock.mockImplementation(() => {
+        throw new Error("Network error");
+      });
+
+      await assert.rejects(() => fetchNewPackagesList(), {
+        message: "Network error",
+      });
+
+      assert.strictEqual(mockFetch.mock.calls.length, 4);
+    });
+
+    it("should return an empty list without fetching for unsupported ecosystems", async () => {
+      ecosystem = "py";
+
+      const result = await fetchNewPackagesList();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 0);
+      assert.deepStrictEqual(result.newPackagesList, []);
+      assert.strictEqual(result.version, undefined);
+    });
+  });
+
+  describe("fetchNewPackagesListVersion", () => {
+    it("should succeed immediately when fetch succeeds on first try", async () => {
+      mockFetch.mock.mockImplementationOnce(() => ({
+        ok: true,
+        headers: { get: () => '"new-packages-etag"' },
+      }));
+
+      const result = await fetchNewPackagesListVersion();
+
+      assert.strictEqual(mockFetch.mock.calls.length, 1);
+      assert.strictEqual(result, '"new-packages-etag"');
+    });
+
+    it("should throw error after exhausting all retries", async () => {
+      mockFetch.mock.mockImplementation(() => {
+        throw new Error("Connection refused");
+      });
+
+      await assert.rejects(() => fetchNewPackagesListVersion(), {
+        message: "Connection refused",
+      });
+
+      assert.strictEqual(mockFetch.mock.calls.length, 4);
+    });
+  });
 });