diff --git a/packages/safe-chain/src/api/aikido.js b/packages/safe-chain/src/api/aikido.js index abb2135..fb01f42 100644 --- a/packages/safe-chain/src/api/aikido.js +++ b/packages/safe-chain/src/api/aikido.js @@ -11,6 +11,13 @@ const malwareDatabaseUrls = { [ECOSYSTEM_PY]: "https://malware-list.aikido.dev/malware_pypi.json", }; +// TODO: replace with the real CDN URL once core publishes the S3 endpoint +const newPackagesListUrls = { + [ECOSYSTEM_JS]: "https://new-packages.aikido.dev/js_packages.json", +}; + +const DEFAULT_FETCH_RETRY_ATTEMPTS = 4; + /** * @typedef {Object} MalwarePackage * @property {string} package_name @@ -18,12 +25,19 @@ const malwareDatabaseUrls = { * @property {string} reason */ +/** + * @typedef {Object} NewPackageEntry + * @property {string} source + * @property {string} name + * @property {string} version + * @property {number} released_on - Unix timestamp (seconds) + * @property {number} scraped_on - Unix timestamp (seconds) + */ + /** * @returns {Promise<{malwareDatabase: MalwarePackage[], version: string | undefined}>} */ export async function fetchMalwareDatabase() { - const numberOfAttempts = 4; - return retry(async () => { const ecosystem = getEcoSystem(); const malwareDatabaseUrl = @@ -46,15 +60,13 @@ export async function fetchMalwareDatabase() { } catch (/** @type {any} */ error) { throw new Error(`Error parsing malware database: ${error.message}`); } - }, numberOfAttempts); + }, DEFAULT_FETCH_RETRY_ATTEMPTS); } /** * @returns {Promise} */ export async function fetchMalwareDatabaseVersion() { - const numberOfAttempts = 4; - return retry(async () => { const ecosystem = getEcoSystem(); const malwareDatabaseUrl = @@ -71,7 +83,63 @@ export async function fetchMalwareDatabaseVersion() { ); } return response.headers.get("etag") || undefined; - }, numberOfAttempts); + }, DEFAULT_FETCH_RETRY_ATTEMPTS); +} + +/** + * @returns {Promise<{newPackagesList: NewPackageEntry[], version: string | undefined}>} + */ +export async function fetchNewPackagesList() { + return retry(async () => { + const ecosystem = getEcoSystem(); + const url = + newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)]; + + if (!url) { + return { newPackagesList: [], version: undefined }; + } + + const response = await fetch(url); + if (!response.ok) { + throw new Error( + `Error fetching ${ecosystem} new packages list: ${response.statusText}` + ); + } + + try { + const newPackagesList = await response.json(); + return { + newPackagesList, + version: response.headers.get("etag") || undefined, + }; + } catch (/** @type {any} */ error) { + throw new Error(`Error parsing new packages list: ${error.message}`); + } + }, DEFAULT_FETCH_RETRY_ATTEMPTS); +} + +/** + * @returns {Promise} + */ +export async function fetchNewPackagesListVersion() { + return retry(async () => { + const ecosystem = getEcoSystem(); + const url = + newPackagesListUrls[/** @type {keyof typeof newPackagesListUrls} */ (ecosystem)]; + + if (!url) { + return undefined; + } + + const response = await fetch(url, { method: "HEAD" }); + if (!response.ok) { + throw new Error( + `Error fetching ${ecosystem} new packages list version: ${response.statusText}` + ); + } + + return response.headers.get("etag") || undefined; + }, DEFAULT_FETCH_RETRY_ATTEMPTS); } /** @@ -91,7 +159,7 @@ async function retry(func, attempts) { return await func(); } catch (error) { ui.writeVerbose( - "An error occurred while trying to download the Aikido Malware database", + "An error occurred while trying to download Aikido data", error ); lastError = error; diff --git a/packages/safe-chain/src/api/aikido.spec.js b/packages/safe-chain/src/api/aikido.spec.js index 2e7cecb..b2d25c2 100644 --- a/packages/safe-chain/src/api/aikido.spec.js +++ b/packages/safe-chain/src/api/aikido.spec.js @@ -3,6 +3,7 @@ import assert from "node:assert"; describe("aikido API", async () => { const mockFetch = mock.fn(); + let ecosystem = "js"; mock.module("make-fetch-happen", { defaultExport: mockFetch, @@ -18,17 +19,22 @@ describe("aikido API", async () => { mock.module("../config/settings.js", { namedExports: { - getEcoSystem: () => "js", + getEcoSystem: () => ecosystem, ECOSYSTEM_JS: "js", ECOSYSTEM_PY: "py", }, }); - const { fetchMalwareDatabase, fetchMalwareDatabaseVersion } = - await import("./aikido.js"); + const { + fetchMalwareDatabase, + fetchMalwareDatabaseVersion, + fetchNewPackagesList, + fetchNewPackagesListVersion, + } = await import("./aikido.js"); beforeEach(() => { mockFetch.mock.resetCalls(); + ecosystem = "js"; }); describe("fetchMalwareDatabase", () => { @@ -130,4 +136,77 @@ describe("aikido API", async () => { assert.strictEqual(result, '"final-etag"'); }); }); + + describe("fetchNewPackagesList", () => { + it("should succeed immediately when fetch succeeds on first try", async () => { + const releases = [ + { + source: "NPM", + name: "fresh-pkg", + version: "1.0.0", + released_on: 123, + scraped_on: 456, + }, + ]; + mockFetch.mock.mockImplementationOnce(() => ({ + ok: true, + json: async () => releases, + headers: { get: () => '"etag-new-packages"' }, + })); + + const result = await fetchNewPackagesList(); + + assert.strictEqual(mockFetch.mock.calls.length, 1); + assert.deepStrictEqual(result.newPackagesList, releases); + assert.strictEqual(result.version, '"etag-new-packages"'); + }); + + it("should throw error after exhausting all retries", async () => { + mockFetch.mock.mockImplementation(() => { + throw new Error("Network error"); + }); + + await assert.rejects(() => fetchNewPackagesList(), { + message: "Network error", + }); + + assert.strictEqual(mockFetch.mock.calls.length, 4); + }); + + it("should return an empty list without fetching for unsupported ecosystems", async () => { + ecosystem = "py"; + + const result = await fetchNewPackagesList(); + + assert.strictEqual(mockFetch.mock.calls.length, 0); + assert.deepStrictEqual(result.newPackagesList, []); + assert.strictEqual(result.version, undefined); + }); + }); + + describe("fetchNewPackagesListVersion", () => { + it("should succeed immediately when fetch succeeds on first try", async () => { + mockFetch.mock.mockImplementationOnce(() => ({ + ok: true, + headers: { get: () => '"new-packages-etag"' }, + })); + + const result = await fetchNewPackagesListVersion(); + + assert.strictEqual(mockFetch.mock.calls.length, 1); + assert.strictEqual(result, '"new-packages-etag"'); + }); + + it("should throw error after exhausting all retries", async () => { + mockFetch.mock.mockImplementation(() => { + throw new Error("Connection refused"); + }); + + await assert.rejects(() => fetchNewPackagesListVersion(), { + message: "Connection refused", + }); + + assert.strictEqual(mockFetch.mock.calls.length, 4); + }); + }); }); diff --git a/packages/safe-chain/src/config/configFile.js b/packages/safe-chain/src/config/configFile.js index bc4dc94..0246fa9 100644 --- a/packages/safe-chain/src/config/configFile.js +++ b/packages/safe-chain/src/config/configFile.js @@ -203,6 +203,70 @@ export function readDatabaseFromLocalCache() { } } +/** + * @param {import("../api/aikido.js").NewPackageEntry[]} data + * @param {string | number} version + * + * @returns {void} + */ +export function writeNewPackagesListToLocalCache(data, version) { + try { + const listPath = getNewPackagesListPath(); + const versionPath = getNewPackagesListVersionPath(); + + fs.writeFileSync(listPath, JSON.stringify(data)); + fs.writeFileSync(versionPath, version.toString()); + } catch { + ui.writeWarning( + "Failed to write new packages list to local cache, next time the list will be fetched from the server again." + ); + } +} + +/** + * @returns {{newPackagesList: import("../api/aikido.js").NewPackageEntry[] | null, version: string | null}} + */ +export function readNewPackagesListFromLocalCache() { + try { + const listPath = getNewPackagesListPath(); + if (!fs.existsSync(listPath)) { + return { newPackagesList: null, version: null }; + } + + const data = fs.readFileSync(listPath, "utf8"); + const newPackagesList = JSON.parse(data); + const versionPath = getNewPackagesListVersionPath(); + let version = null; + if (fs.existsSync(versionPath)) { + version = fs.readFileSync(versionPath, "utf8").trim(); + } + return { newPackagesList, version }; + } catch { + ui.writeWarning( + "Failed to read new packages list from local cache. Continuing without local cache." + ); + return { newPackagesList: null, version: null }; + } +} + +/** + * @returns {string} + */ +function getNewPackagesListPath() { + const safeChainDir = getSafeChainDirectory(); + const ecosystem = getEcoSystem(); + return path.join(safeChainDir, `newPackagesList_${ecosystem}.json`); +} + +/** + * @returns {string} + */ +function getNewPackagesListVersionPath() { + const safeChainDir = getSafeChainDirectory(); + const ecosystem = getEcoSystem(); + return path.join(safeChainDir, `newPackagesList_version_${ecosystem}.txt`); +} + /** * @returns {SafeChainConfig} */ diff --git a/packages/safe-chain/src/scanning/newPackagesDatabase.js b/packages/safe-chain/src/scanning/newPackagesDatabase.js new file mode 100644 index 0000000..31afb7d --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesDatabase.js @@ -0,0 +1,112 @@ +import { + fetchNewPackagesList, + fetchNewPackagesListVersion, +} from "../api/aikido.js"; +import { + readNewPackagesListFromLocalCache, + writeNewPackagesListToLocalCache, +} from "../config/configFile.js"; +import { ui } from "../environment/userInteraction.js"; +import { + getMinimumPackageAgeHours, + getEcoSystem, + ECOSYSTEM_JS, +} from "../config/settings.js"; + +/** + * @typedef {Object} NewPackagesDatabase + * @property {function(string, string): boolean} isNewlyReleasedPackage + */ + +/** @type {NewPackagesDatabase | null} */ +let cachedNewPackagesDatabase = null; + +/** + * Returns the source identifier used in the feed for the current ecosystem. + * @returns {string} + */ +function getCurrentFeedSource() { + return getEcoSystem(); +} + +/** + * @returns {Promise} + */ +export async function openNewPackagesDatabase() { + if (cachedNewPackagesDatabase) { + return cachedNewPackagesDatabase; + } + + if (getEcoSystem() !== ECOSYSTEM_JS) { + cachedNewPackagesDatabase = { isNewlyReleasedPackage: () => false }; + return cachedNewPackagesDatabase; + } + + const newPackagesList = await getNewPackagesList(); + + /** + * @param {string} name + * @param {string} version + * @returns {boolean} + */ + function isNewlyReleasedPackage(name, version) { + const cutOff = new Date( + new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000 + ); + const expectedSource = getCurrentFeedSource(); + + const entry = newPackagesList.find( + (pkg) => + pkg.source?.toLowerCase() === expectedSource && + pkg.name === name && + pkg.version === version + ); + + if (!entry) { + return false; + } + + const releasedOn = new Date(entry.released_on * 1000); + return releasedOn > cutOff; + } + + cachedNewPackagesDatabase = { isNewlyReleasedPackage }; + return cachedNewPackagesDatabase; +} + +/** + * @returns {Promise} + */ +async function getNewPackagesList() { + const { newPackagesList: cachedList, version: cachedVersion } = + readNewPackagesListFromLocalCache(); + + try { + if (cachedList) { + const currentVersion = await fetchNewPackagesListVersion(); + if (cachedVersion === currentVersion) { + return cachedList; + } + } + + const { newPackagesList, version } = await fetchNewPackagesList(); + + if (version) { + writeNewPackagesListToLocalCache(newPackagesList, version); + return newPackagesList; + } else { + ui.writeWarning( + "The new packages list was downloaded, but could not be cached due to a missing version." + ); + return newPackagesList; + } + } catch (/** @type {any} */ error) { + if (cachedList) { + ui.writeWarning( + "Failed to fetch the latest new packages list. Using cached version." + ); + return cachedList; + } + throw error; + } +} diff --git a/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js b/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js new file mode 100644 index 0000000..60a806f --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js @@ -0,0 +1,230 @@ +import { describe, it, mock, beforeEach } from "node:test"; +import assert from "node:assert"; + +// --- shared mutable state for mocks --- +let cachedList = null; +let cachedVersion = null; +let fetchedList = []; +let fetchedVersion = "etag-1"; +let fetchVersionResult = "etag-1"; +let minimumPackageAgeHours = 24; +let ecosystem = "js"; +let writeWarningCalls = []; +let fetchListError = null; +let fetchVersionError = null; +let importCounter = 0; + +mock.module("../api/aikido.js", { + namedExports: { + fetchNewPackagesList: async () => { + if (fetchListError) { + throw fetchListError; + } + + return { + newPackagesList: fetchedList, + version: fetchedVersion, + }; + }, + fetchNewPackagesListVersion: async () => { + if (fetchVersionError) { + throw fetchVersionError; + } + + return fetchVersionResult; + }, + }, +}); + +mock.module("../config/configFile.js", { + namedExports: { + readNewPackagesListFromLocalCache: () => ({ + newPackagesList: cachedList, + version: cachedVersion, + }), + writeNewPackagesListToLocalCache: () => {}, + }, +}); + +mock.module("../environment/userInteraction.js", { + namedExports: { + ui: { + writeWarning: (msg) => writeWarningCalls.push(msg), + writeVerbose: () => {}, + }, + }, +}); + +mock.module("../config/settings.js", { + namedExports: { + getMinimumPackageAgeHours: () => minimumPackageAgeHours, + getEcoSystem: () => ecosystem, + ECOSYSTEM_JS: "js", + ECOSYSTEM_PY: "py", + }, +}); + +describe("newPackagesDatabase", async () => { + beforeEach(() => { + cachedList = null; + cachedVersion = null; + fetchedList = []; + fetchedVersion = "etag-1"; + fetchVersionResult = "etag-1"; + minimumPackageAgeHours = 24; + ecosystem = "js"; + writeWarningCalls = []; + fetchListError = null; + fetchVersionError = null; + }); + + async function openNewPackagesDatabase() { + const module = await import( + `./newPackagesDatabase.js?test_case=${importCounter++}` + ); + return module.openNewPackagesDatabase(); + } + + function hoursAgo(hours) { + return Math.floor((Date.now() - hours * 3600 * 1000) / 1000); + } + + describe("isNewlyReleasedPackage", () => { + it("returns true for a package released within the age threshold", async () => { + fetchedList = [ + { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + ]; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + }); + + it("returns false for a package released outside the age threshold", async () => { + fetchedList = [ + { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(48), scraped_on: hoursAgo(48) }, + ]; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + }); + + it("returns false for a package not in the list", async () => { + fetchedList = []; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("not-there", "1.0.0"), false); + }); + + it("returns false for a known package but different version", async () => { + fetchedList = [ + { source: "js", name: "foo", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + ]; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + }); + + it("ignores entries from a different source in a mixed feed", async () => { + fetchedList = [ + { + source: "npm", + name: "foo", + version: "1.0.0", + released_on: hoursAgo(1), + scraped_on: hoursAgo(1), + }, + { + source: "js", + name: "bar", + version: "1.0.0", + released_on: hoursAgo(1), + scraped_on: hoursAgo(1), + }, + ]; + + const db = await openNewPackagesDatabase(); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + assert.strictEqual(db.isNewlyReleasedPackage("bar", "1.0.0"), true); + }); + + it("respects a custom minimumPackageAgeHours threshold", async () => { + minimumPackageAgeHours = 168; // 7 days + fetchedList = [ + { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(100), scraped_on: hoursAgo(100) }, + ]; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + }); + + it("returns false for all packages when ecosystem is not JS", async () => { + ecosystem = "py"; + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + }); + }); + + describe("caching behaviour", () => { + it("uses local cache when etag matches", async () => { + cachedList = [ + { source: "js", name: "cached-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + ]; + cachedVersion = "etag-1"; + fetchVersionResult = "etag-1"; + // fetchedList is empty — if we used the remote list, the lookup would return false + fetchedList = []; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("cached-pkg", "1.0.0"), true); + }); + + it("fetches fresh list when etag does not match", async () => { + cachedList = [ + { source: "js", name: "stale-pkg", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + ]; + cachedVersion = "etag-old"; + fetchVersionResult = "etag-new"; + fetchedList = [ + { source: "js", name: "fresh-pkg", version: "2.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + ]; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("stale-pkg", "1.0.0"), false); + assert.strictEqual(db.isNewlyReleasedPackage("fresh-pkg", "2.0.0"), true); + }); + + it("falls back to local cache when fetch fails", async () => { + cachedList = [ + { + source: "js", + name: "cached-pkg", + version: "1.0.0", + released_on: hoursAgo(1), + scraped_on: hoursAgo(1), + }, + ]; + cachedVersion = "etag-old"; + fetchVersionResult = "etag-new"; + fetchListError = new Error("Network error"); + + const db = await openNewPackagesDatabase(); + + assert.strictEqual(db.isNewlyReleasedPackage("cached-pkg", "1.0.0"), true); + assert.strictEqual(writeWarningCalls.length, 1); + assert.ok(writeWarningCalls[0].includes("Using cached version")); + }); + + it("emits a warning when list has no version (cannot be cached)", async () => { + fetchedList = [ + { source: "js", name: "foo", version: "1.0.0", released_on: hoursAgo(1), scraped_on: hoursAgo(1) }, + ]; + fetchedVersion = undefined; + + const db = await openNewPackagesDatabase(); + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + assert.strictEqual(writeWarningCalls.length, 1); + assert.ok(writeWarningCalls[0].includes("could not be cached")); + }); + }); +}); diff --git a/packages/safe-chain/src/ultimate/ultimateTroubleshooting.js b/packages/safe-chain/src/ultimate/ultimateTroubleshooting.js index e333615..114bd5e 100644 --- a/packages/safe-chain/src/ultimate/ultimateTroubleshooting.js +++ b/packages/safe-chain/src/ultimate/ultimateTroubleshooting.js @@ -42,7 +42,7 @@ export async function troubleshootingExport() { resolve(zipFileName); }); - archive.on('error', (err) => { + archive.on('error', (/** @type {Error} */ err) => { ui.writeError(`Failed to zip logs: ${err.message}`); reject(err); });