From ac09534070efb2e34b76fd4650b1675044198c53 Mon Sep 17 00:00:00 2001 From: Reinier Criel Date: Fri, 20 Mar 2026 09:11:02 -0700 Subject: [PATCH] Adapt per latest core --- packages/safe-chain/src/api/aikido.js | 8 ++--- packages/safe-chain/src/api/aikido.spec.js | 5 ++-- .../src/scanning/newPackagesDatabase.js | 19 +++++++++--- .../src/scanning/newPackagesDatabase.spec.js | 29 +++++++++---------- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/packages/safe-chain/src/api/aikido.js b/packages/safe-chain/src/api/aikido.js index fb01f42..5248e0f 100644 --- a/packages/safe-chain/src/api/aikido.js +++ b/packages/safe-chain/src/api/aikido.js @@ -11,9 +11,9 @@ const malwareDatabaseUrls = { [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/malware_pypi.json", }; -// TODO: replace with the real CDN URL once core publishes the S3 endpoint const newPackagesListUrls = { - [ECOSYSTEM_JS]: "https://new-packages.aikido.dev/js_packages.json", + [ECOSYSTEM_JS]: "https://malware-list.aikido.dev/releases_npm.json", + [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/releases_pypi.json", }; const DEFAULT_FETCH_RETRY_ATTEMPTS = 4; @@ -27,8 +27,8 @@ const DEFAULT_FETCH_RETRY_ATTEMPTS = 4; /** * @typedef {Object} NewPackageEntry - * @property {string} source - * @property {string} name + * @property {string} [source] + * @property {string} package_name * @property {string} version * @property {number} released_on - Unix timestamp (seconds) * @property {number} scraped_on - Unix timestamp (seconds) diff --git a/packages/safe-chain/src/api/aikido.spec.js b/packages/safe-chain/src/api/aikido.spec.js index b2d25c2..d70f7e2 100644 --- a/packages/safe-chain/src/api/aikido.spec.js +++ b/packages/safe-chain/src/api/aikido.spec.js @@ -141,8 +141,7 @@ describe("aikido API", async () => { it("should succeed immediately when fetch succeeds on first try", async () => { const releases = [ { - source: "NPM", - name: "fresh-pkg", + package_name: "fresh-pkg", version: "1.0.0", released_on: 123, scraped_on: 456, @@ -174,7 +173,7 @@ describe("aikido API", async () => { }); it("should return an empty list without fetching for unsupported ecosystems", async () => { - ecosystem = "py"; + ecosystem = "ruby"; const result = await fetchNewPackagesList(); diff --git a/packages/safe-chain/src/scanning/newPackagesDatabase.js b/packages/safe-chain/src/scanning/newPackagesDatabase.js index fb99164..b480dab 100644 --- a/packages/safe-chain/src/scanning/newPackagesDatabase.js +++ b/packages/safe-chain/src/scanning/newPackagesDatabase.js @@ -11,6 +11,7 @@ import { getMinimumPackageAgeHours, getEcoSystem, ECOSYSTEM_JS, + ECOSYSTEM_PY, } from "../config/settings.js"; /** @@ -23,11 +24,21 @@ let cachedNewPackagesDatabase = null; let hasWarnedAboutUnavailableNewPackagesDatabase = false; /** - * Returns the source identifier used in the feed for the current ecosystem. + * Returns the ecosystem identifier expected in upstream/core release feeds. * @returns {string} */ function getCurrentFeedSource() { - return getEcoSystem(); + const ecosystem = getEcoSystem(); + + if (ecosystem === ECOSYSTEM_JS) { + return "npm"; + } + + if (ecosystem === ECOSYSTEM_PY) { + return "pypi"; + } + + return ecosystem; } /** @@ -73,8 +84,8 @@ export async function openNewPackagesDatabase() { const entry = newPackagesList.find( (pkg) => - pkg.source?.toLowerCase() === expectedSource && - pkg.name === name && + (!pkg.source || pkg.source.toLowerCase() === expectedSource) && + pkg.package_name === name && pkg.version === version ); diff --git a/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js b/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js index e2c88f7..58c9a74 100644 --- a/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js +++ b/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js @@ -96,7 +96,7 @@ describe("newPackagesDatabase", async () => { describe("isNewlyReleasedPackage", () => { it("returns true for a package released within the age threshold", async () => { fetchedList = [ - { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, ]; const db = await openNewPackagesDatabase(); @@ -105,7 +105,7 @@ describe("newPackagesDatabase", async () => { it("returns false for a package released outside the age threshold", async () => { fetchedList = [ - { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(48), scraped_on: hoursAgo(48) }, + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(48), scraped_on: hoursAgo(48) }, ]; const db = await openNewPackagesDatabase(); @@ -121,25 +121,25 @@ describe("newPackagesDatabase", async () => { it("returns false for a known package but different version", async () => { fetchedList = [ - { source: "js", name: "foo", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + { package_name: "foo", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, ]; const db = await openNewPackagesDatabase(); assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); }); - it("ignores entries from a different source in a mixed feed", async () => { + it("matches the current feed ecosystem when source metadata is present", async () => { fetchedList = [ { - source: "npm", - name: "foo", + source: "pypi", + package_name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1), }, { - source: "js", - name: "bar", + source: "npm", + package_name: "bar", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1), @@ -155,7 +155,7 @@ describe("newPackagesDatabase", async () => { it("respects a custom minimumPackageAgeHours threshold", async () => { minimumPackageAgeHours = 168; // 7 days fetchedList = [ - { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(100), scraped_on: hoursAgo(100) }, + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(100), scraped_on: hoursAgo(100) }, ]; const db = await openNewPackagesDatabase(); @@ -172,7 +172,7 @@ describe("newPackagesDatabase", async () => { describe("caching behaviour", () => { it("uses local cache when etag matches", async () => { cachedList = [ - { source: "js", name: "cached-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + { package_name: "cached-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, ]; cachedVersion = "etag-1"; fetchVersionResult = "etag-1"; @@ -185,12 +185,12 @@ describe("newPackagesDatabase", async () => { it("fetches fresh list when etag does not match", async () => { cachedList = [ - { source: "js", name: "stale-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + { package_name: "stale-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, ]; cachedVersion = "etag-old"; fetchVersionResult = "etag-new"; fetchedList = [ - { source: "js", name: "fresh-pkg", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + { package_name: "fresh-pkg", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, ]; const db = await openNewPackagesDatabase(); @@ -201,8 +201,7 @@ describe("newPackagesDatabase", async () => { it("falls back to local cache when fetch fails", async () => { cachedList = [ { - source: "js", - name: "cached-pkg", + package_name: "cached-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1), @@ -221,7 +220,7 @@ describe("newPackagesDatabase", async () => { it("emits a warning when list has no version (cannot be cached)", async () => { fetchedList = [ - { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, ]; fetchedVersion = undefined;