diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js index 2a41524..f4e4e1b 100644 --- a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js @@ -13,7 +13,7 @@ import { modifyNpmInfoResponse, } from "./modifyNpmInfo.js"; import { parseNpmPackageUrl } from "./parseNpmPackageUrl.js"; -import { openNewPackagesDatabase } from "../../../scanning/newPackagesDatabase.js"; +import { openNewPackagesDatabase } from "../../../scanning/newPackagesListCache.js"; const knownJsRegistries = [ "registry.npmjs.org", diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js index 45d3ceb..de7acc6 100644 --- a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js @@ -18,7 +18,7 @@ describe("npmInterceptor minimum package age", async () => { getEcoSystem: () => "js", }, }); - mock.module("../../../scanning/newPackagesDatabase.js", { + mock.module("../../../scanning/newPackagesListCache.js", { namedExports: { openNewPackagesDatabase: async () => ({ isNewlyReleasedPackage: (name, version) => diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.packageDownload.spec.js b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.packageDownload.spec.js index 0c4b377..e361275 100644 --- a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.packageDownload.spec.js +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.packageDownload.spec.js @@ -32,7 +32,7 @@ mock.module("../../../config/settings.js", { skipMinimumPackageAge: () => skipMinimumPackageAgeSetting, }, }); -mock.module("../../../scanning/newPackagesDatabase.js", { +mock.module("../../../scanning/newPackagesListCache.js", { namedExports: { openNewPackagesDatabase: async () => ({ isNewlyReleasedPackage: (name, version) => diff --git a/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js b/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js index e83df62..902f705 100644 --- a/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js +++ b/packages/safe-chain/src/scanning/newPackagesDatabase.spec.js @@ -56,6 +56,9 @@ mock.module("../config/settings.js", { }, }); +// Import the warnings module so we can reset its state between tests. +const { resetWarningState } = await import("./newPackagesDatabaseWarnings.js"); + describe("newPackagesDatabase", async () => { beforeEach(() => { fetchedList = []; @@ -66,6 +69,7 @@ describe("newPackagesDatabase", async () => { writeWarningCalls = []; fetchListError = null; fetchVersionError = null; + resetWarningState(); testHomeDir = path.join( os.tmpdir(), `safe-chain-new-packages-db-${process.pid}-${importCounter}` @@ -77,13 +81,13 @@ describe("newPackagesDatabase", async () => { async function openNewPackagesDatabase() { const module = await import( - `./newPackagesDatabase.js?test_case=${importCounter++}` + `./newPackagesListCache.js?test_case=${importCounter++}` ); return module.openNewPackagesDatabase(); } async function loadNewPackagesDatabaseModule() { - return import(`./newPackagesDatabase.js?test_case=${importCounter++}`); + return import(`./newPackagesListCache.js?test_case=${importCounter++}`); } function hoursAgo(hours) { diff --git a/packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.js b/packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.js new file mode 100644 index 0000000..6db4a66 --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.js @@ -0,0 +1,63 @@ +import { + getMinimumPackageAgeHours, + getEcoSystem, + ECOSYSTEM_JS, + ECOSYSTEM_PY, +} from "../config/settings.js"; + +/** + * @typedef {Object} NewPackagesDatabase + * @property {function(string, string): boolean} isNewlyReleasedPackage + */ + +/** + * Returns the ecosystem identifier expected in upstream/core release feeds. + * @returns {string} + */ +function getCurrentFeedSource() { + const ecosystem = getEcoSystem(); + + if (ecosystem === ECOSYSTEM_JS) { + return "npm"; + } + + if (ecosystem === ECOSYSTEM_PY) { + return "pypi"; + } + + return ecosystem; +} + +/** + * @param {import("../api/aikido.js").NewPackageEntry[]} newPackagesList + * @returns {NewPackagesDatabase} + */ +export function buildNewPackagesDatabase(newPackagesList) { + /** + * @param {string} name + * @param {string} version + * @returns {boolean} + */ + function isNewlyReleasedPackage(name, version) { + const cutOff = new Date( + new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000 + ); + const expectedSource = getCurrentFeedSource(); + + const entry = newPackagesList.find( + (pkg) => + (!pkg.source || pkg.source.toLowerCase() === expectedSource) && + pkg.package_name === name && + pkg.version === version + ); + + if (!entry) { + return false; + } + + const releasedOn = new Date(entry.released_on * 1000); + return releasedOn > cutOff; + } + + return { isNewlyReleasedPackage }; +} diff --git a/packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.spec.js b/packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.spec.js new file mode 100644 index 0000000..0c2fb84 --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesDatabaseBuilder.spec.js @@ -0,0 +1,100 @@ +import { describe, it, mock } from "node:test"; +import assert from "node:assert"; + +let minimumPackageAgeHours = 24; +let ecosystem = "js"; + +mock.module("../config/settings.js", { + namedExports: { + getMinimumPackageAgeHours: () => minimumPackageAgeHours, + getEcoSystem: () => ecosystem, + ECOSYSTEM_JS: "js", + ECOSYSTEM_PY: "py", + }, +}); + +const { buildNewPackagesDatabase } = await import( + "./newPackagesDatabaseBuilder.js" +); + +function hoursAgo(hours) { + return Math.floor((Date.now() - hours * 3600 * 1000) / 1000); +} + +describe("buildNewPackagesDatabase", () => { + it("returns an object with isNewlyReleasedPackage", () => { + const db = buildNewPackagesDatabase([]); + assert.strictEqual(typeof db.isNewlyReleasedPackage, "function"); + }); + + describe("isNewlyReleasedPackage", () => { + it("returns true for a package released within the age threshold", () => { + const db = buildNewPackagesDatabase([ + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) }, + ]); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + }); + + it("returns false for a package released outside the age threshold", () => { + const db = buildNewPackagesDatabase([ + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(48) }, + ]); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + }); + + it("returns false for a package not in the list", () => { + const db = buildNewPackagesDatabase([]); + + assert.strictEqual(db.isNewlyReleasedPackage("not-there", "1.0.0"), false); + }); + + it("returns false for a known package but different version", () => { + const db = buildNewPackagesDatabase([ + { package_name: "foo", version: "2.0.0", released_on: hoursAgo(1) }, + ]); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + }); + + it("filters by source when source metadata is present", () => { + const db = buildNewPackagesDatabase([ + { source: "pypi", package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) }, + { source: "npm", package_name: "bar", version: "1.0.0", released_on: hoursAgo(1) }, + ]); + + // ecosystem is "js" → feed source is "npm" + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false); + assert.strictEqual(db.isNewlyReleasedPackage("bar", "1.0.0"), true); + }); + + it("matches regardless of source case", () => { + const db = buildNewPackagesDatabase([ + { source: "NPM", package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) }, + ]); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + }); + + it("matches entries with no source field", () => { + const db = buildNewPackagesDatabase([ + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) }, + ]); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + }); + + it("respects a custom minimumPackageAgeHours threshold", () => { + minimumPackageAgeHours = 168; // 7 days + + const db = buildNewPackagesDatabase([ + { package_name: "foo", version: "1.0.0", released_on: hoursAgo(100) }, + ]); + + assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true); + + minimumPackageAgeHours = 24; // reset + }); + }); +}); diff --git a/packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.js b/packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.js new file mode 100644 index 0000000..fd742bb --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.js @@ -0,0 +1,17 @@ +import { ui } from "../environment/userInteraction.js"; + +let hasWarnedAboutUnavailableNewPackagesDatabase = false; + +/** @param {Error} error */ +export function warnOnceAboutUnavailableDatabase(error) { + if (!hasWarnedAboutUnavailableNewPackagesDatabase) { + ui.writeWarning( + `Failed to load the new packages list used for direct package download request blocking. Continuing with metadata-based minimum age checks only. ${error.message}` + ); + hasWarnedAboutUnavailableNewPackagesDatabase = true; + } +} + +export function resetWarningState() { + hasWarnedAboutUnavailableNewPackagesDatabase = false; +} diff --git a/packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.spec.js b/packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.spec.js new file mode 100644 index 0000000..d36d5df --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesDatabaseWarnings.spec.js @@ -0,0 +1,63 @@ +import { describe, it, mock, beforeEach } from "node:test"; +import assert from "node:assert"; + +let writeWarningCalls = []; + +mock.module("../environment/userInteraction.js", { + namedExports: { + ui: { + writeWarning: (msg) => writeWarningCalls.push(msg), + }, + }, +}); + +const { warnOnceAboutUnavailableDatabase, resetWarningState } = await import( + "./newPackagesDatabaseWarnings.js" +); + +describe("newPackagesDatabaseWarnings", () => { + beforeEach(() => { + writeWarningCalls = []; + resetWarningState(); + }); + + describe("warnOnceAboutUnavailableDatabase", () => { + it("emits a warning containing the error message", () => { + warnOnceAboutUnavailableDatabase(new Error("feed unavailable")); + + assert.strictEqual(writeWarningCalls.length, 1); + assert.ok(writeWarningCalls[0].includes("feed unavailable")); + }); + + it("mentions fallback to metadata-based checks in the warning", () => { + warnOnceAboutUnavailableDatabase(new Error("timeout")); + + assert.ok( + writeWarningCalls[0].includes( + "Continuing with metadata-based minimum age checks only" + ) + ); + }); + + it("only emits once even when called multiple times", () => { + warnOnceAboutUnavailableDatabase(new Error("first")); + warnOnceAboutUnavailableDatabase(new Error("second")); + warnOnceAboutUnavailableDatabase(new Error("third")); + + assert.strictEqual(writeWarningCalls.length, 1); + }); + }); + + describe("resetWarningState", () => { + it("allows the warning to fire again after reset", () => { + warnOnceAboutUnavailableDatabase(new Error("first")); + assert.strictEqual(writeWarningCalls.length, 1); + + resetWarningState(); + writeWarningCalls = []; + + warnOnceAboutUnavailableDatabase(new Error("second")); + assert.strictEqual(writeWarningCalls.length, 1); + }); + }); +}); diff --git a/packages/safe-chain/src/scanning/newPackagesDatabase.js b/packages/safe-chain/src/scanning/newPackagesListCache.js similarity index 68% rename from packages/safe-chain/src/scanning/newPackagesDatabase.js rename to packages/safe-chain/src/scanning/newPackagesListCache.js index 6a74656..f7496b6 100644 --- a/packages/safe-chain/src/scanning/newPackagesDatabase.js +++ b/packages/safe-chain/src/scanning/newPackagesListCache.js @@ -8,40 +8,17 @@ import { getNewPackagesListVersionPath, } from "../config/configFile.js"; import { ui } from "../environment/userInteraction.js"; -import { - getMinimumPackageAgeHours, - getEcoSystem, - ECOSYSTEM_JS, - ECOSYSTEM_PY, -} from "../config/settings.js"; +import { getEcoSystem, ECOSYSTEM_JS } from "../config/settings.js"; +import { buildNewPackagesDatabase } from "./newPackagesDatabaseBuilder.js"; +import { warnOnceAboutUnavailableDatabase } from "./newPackagesDatabaseWarnings.js"; /** - * @typedef {Object} NewPackagesDatabase - * @property {function(string, string): boolean} isNewlyReleasedPackage + * @typedef {import("./newPackagesDatabaseBuilder.js").NewPackagesDatabase} NewPackagesDatabase */ // Shared per-process cache to avoid rebuilding the same feed-backed database on each request. /** @type {NewPackagesDatabase | null} */ let cachedNewPackagesDatabase = null; -let hasWarnedAboutUnavailableNewPackagesDatabase = false; - -/** - * Returns the ecosystem identifier expected in upstream/core release feeds. - * @returns {string} - */ -function getCurrentFeedSource() { - const ecosystem = getEcoSystem(); - - if (ecosystem === ECOSYSTEM_JS) { - return "npm"; - } - - if (ecosystem === ECOSYSTEM_PY) { - return "pypi"; - } - - return ecosystem; -} /** * @returns {Promise} @@ -62,44 +39,12 @@ export async function openNewPackagesDatabase() { try { newPackagesList = await getNewPackagesList(); } catch (/** @type {any} */ error) { - if (!hasWarnedAboutUnavailableNewPackagesDatabase) { - ui.writeWarning( - `Failed to load the new packages list used for direct package download request blocking. Continuing with metadata-based minimum age checks only. ${error.message}` - ); - hasWarnedAboutUnavailableNewPackagesDatabase = true; - } - + warnOnceAboutUnavailableDatabase(error); cachedNewPackagesDatabase = { isNewlyReleasedPackage: () => false }; return cachedNewPackagesDatabase; } - /** - * @param {string} name - * @param {string} version - * @returns {boolean} - */ - function isNewlyReleasedPackage(name, version) { - const cutOff = new Date( - new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000 - ); - const expectedSource = getCurrentFeedSource(); - - const entry = newPackagesList.find( - (pkg) => - (!pkg.source || pkg.source.toLowerCase() === expectedSource) && - pkg.package_name === name && - pkg.version === version - ); - - if (!entry) { - return false; - } - - const releasedOn = new Date(entry.released_on * 1000); - return releasedOn > cutOff; - } - - cachedNewPackagesDatabase = { isNewlyReleasedPackage }; + cachedNewPackagesDatabase = buildNewPackagesDatabase(newPackagesList); return cachedNewPackagesDatabase; } diff --git a/packages/safe-chain/src/scanning/newPackagesListCache.spec.js b/packages/safe-chain/src/scanning/newPackagesListCache.spec.js new file mode 100644 index 0000000..8616876 --- /dev/null +++ b/packages/safe-chain/src/scanning/newPackagesListCache.spec.js @@ -0,0 +1,177 @@ +import { describe, it, mock, beforeEach } from "node:test"; +import assert from "node:assert"; +import fs from "fs"; +import path from "path"; +import os from "os"; + +let writeWarningCalls = []; +let ecosystem = "js"; +let testHomeDir = ""; + +mock.module("../environment/userInteraction.js", { + namedExports: { + ui: { + writeWarning: (msg) => writeWarningCalls.push(msg), + }, + }, +}); + +mock.module("../config/settings.js", { + namedExports: { + getEcoSystem: () => ecosystem, + getMinimumPackageAgeHours: () => 24, + ECOSYSTEM_JS: "js", + ECOSYSTEM_PY: "py", + }, +}); + +const { readNewPackagesListFromLocalCache, writeNewPackagesListToLocalCache } = + await import("./newPackagesListCache.js"); + +describe("newPackagesListCache", () => { + beforeEach(() => { + writeWarningCalls = []; + ecosystem = "js"; + testHomeDir = path.join( + os.tmpdir(), + `safe-chain-list-cache-${process.pid}-${Date.now()}` + ); + fs.rmSync(testHomeDir, { recursive: true, force: true }); + fs.mkdirSync(testHomeDir, { recursive: true }); + process.env.HOME = testHomeDir; + }); + + describe("readNewPackagesListFromLocalCache", () => { + it("returns null for both fields when no cache file exists", () => { + const result = readNewPackagesListFromLocalCache(); + + assert.deepStrictEqual(result, { newPackagesList: null, version: null }); + }); + + it("returns the list and version when both files exist", () => { + const list = [{ package_name: "foo", version: "1.0.0" }]; + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_js.json"), + JSON.stringify(list) + ); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_version_js.txt"), + "etag-42" + ); + + const result = readNewPackagesListFromLocalCache(); + + assert.deepStrictEqual(result.newPackagesList, list); + assert.strictEqual(result.version, "etag-42"); + }); + + it("returns null version when version file is missing", () => { + const list = [{ package_name: "foo", version: "1.0.0" }]; + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_js.json"), + JSON.stringify(list) + ); + + const result = readNewPackagesListFromLocalCache(); + + assert.deepStrictEqual(result.newPackagesList, list); + assert.strictEqual(result.version, null); + }); + + it("trims whitespace from the version string", () => { + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_js.json"), + JSON.stringify([]) + ); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_version_js.txt"), + " etag-trimmed \n" + ); + + const { version } = readNewPackagesListFromLocalCache(); + + assert.strictEqual(version, "etag-trimmed"); + }); + + it("uses the ecosystem name in the file path", () => { + ecosystem = "py"; + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_py.json"), + JSON.stringify([{ package_name: "requests", version: "2.0.0" }]) + ); + + const result = readNewPackagesListFromLocalCache(); + + assert.ok(result.newPackagesList !== null); + }); + + it("warns and returns nulls when the list file contains invalid JSON", () => { + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + fs.writeFileSync( + path.join(safeChainDir, "newPackagesList_js.json"), + "not-valid-json" + ); + + const result = readNewPackagesListFromLocalCache(); + + assert.deepStrictEqual(result, { newPackagesList: null, version: null }); + assert.strictEqual(writeWarningCalls.length, 1); + assert.ok(writeWarningCalls[0].includes("local cache")); + }); + }); + + describe("writeNewPackagesListToLocalCache", () => { + it("writes the list and version to disk", () => { + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + + const list = [{ package_name: "foo", version: "1.0.0" }]; + writeNewPackagesListToLocalCache(list, "etag-99"); + + const writtenList = JSON.parse( + fs.readFileSync(path.join(safeChainDir, "newPackagesList_js.json"), "utf8") + ); + const writtenVersion = fs.readFileSync( + path.join(safeChainDir, "newPackagesList_version_js.txt"), + "utf8" + ); + + assert.deepStrictEqual(writtenList, list); + assert.strictEqual(writtenVersion, "etag-99"); + }); + + it("converts a numeric version to a string", () => { + const safeChainDir = path.join(testHomeDir, ".safe-chain"); + fs.mkdirSync(safeChainDir, { recursive: true }); + + writeNewPackagesListToLocalCache([], 42); + + const written = fs.readFileSync( + path.join(safeChainDir, "newPackagesList_version_js.txt"), + "utf8" + ); + assert.strictEqual(written, "42"); + }); + + it("warns when writing fails", () => { + // Place a regular file at the .safe-chain path so getSafeChainDirectory + // returns it as-is (existsSync is true) but writing a child path fails. + const safeChainPath = path.join(testHomeDir, ".safe-chain"); + fs.writeFileSync(safeChainPath, "not-a-directory"); + + writeNewPackagesListToLocalCache([], "etag-fail"); + + assert.strictEqual(writeWarningCalls.length, 1); + assert.ok(writeWarningCalls[0].includes("local cache")); + }); + }); +});