From 6ae93686b7a06a622f505021cd44b114cb58b522 Mon Sep 17 00:00:00 2001 From: Sander Declerck Date: Thu, 13 Nov 2025 14:51:57 +0100 Subject: [PATCH] Finish npm info modification. --- packages/safe-chain/src/config/settings.js | 3 +- .../interceptors/interceptorBuilder.js | 54 ++++- .../interceptors/npm/modifyNpmInfo.js | 148 +++++++++++++ .../interceptors/npm/npmInterceptor.js | 197 +----------------- .../npm/npmInterceptor.minPackageAge.spec.js | 29 ++- .../interceptors/npm/parseNpmPackageUrl.js | 43 ++++ .../src/registryProxy/mitmRequestHandler.js | 19 +- 7 files changed, 281 insertions(+), 212 deletions(-) create mode 100644 packages/safe-chain/src/registryProxy/interceptors/npm/modifyNpmInfo.js create mode 100644 packages/safe-chain/src/registryProxy/interceptors/npm/parseNpmPackageUrl.js diff --git a/packages/safe-chain/src/config/settings.js b/packages/safe-chain/src/config/settings.js index 5b27118..cef66c3 100644 --- a/packages/safe-chain/src/config/settings.js +++ b/packages/safe-chain/src/config/settings.js @@ -37,6 +37,7 @@ export function setEcoSystem(setting) { ecosystemSettings.ecoSystem = setting; } +const defaultMinimumPackageAge = 24; export function getMinimumPackageAgeHours() { - return 24 * 6; + return defaultMinimumPackageAge; } diff --git a/packages/safe-chain/src/registryProxy/interceptors/interceptorBuilder.js b/packages/safe-chain/src/registryProxy/interceptors/interceptorBuilder.js index 96c1e67..c8ef3fc 100644 --- a/packages/safe-chain/src/registryProxy/interceptors/interceptorBuilder.js +++ b/packages/safe-chain/src/registryProxy/interceptors/interceptorBuilder.js @@ -10,11 +10,16 @@ import { EventEmitter } from "events"; * @typedef {Object} RequestInterceptionContext * @property {string} targetUrl * @property {(packageName: string | undefined, version: string | undefined) => void} blockMalware + * @property {(modificationFunc: (headers: NodeJS.Dict) => void) => void} modifyRequestHeaders + * @property {(modificationFunc: (body: Buffer) => Buffer) => void} modifyBody * @property {() => RequestInterceptionHandler} build * * * @typedef {Object} RequestInterceptionHandler * @property {{statusCode: number, message: string} | undefined} blockResponse + * @property {(headers: NodeJS.Dict | undefined) => void} modifyRequestHeaders + * @property {() => boolean} modifiesResponse + * @property {(body: Buffer) => Buffer} modifyBody */ /** @@ -60,12 +65,16 @@ function buildInterceptor(requestHandlers) { function createRequestContext(targetUrl, eventEmitter) { /** @type {{statusCode: number, message: string} | undefined} */ let blockResponse = undefined; + /** @type {Array<(headers: NodeJS.Dict) => void>} */ + let reqheaderModificationFuncs = []; + /** @type {Array<(body: Buffer) => Buffer>} */ + let modifyBodyFuncs = []; /** * @param {string | undefined} packageName * @param {string | undefined} version */ - function blockMalware(packageName, version) { + function blockMalwareSetup(packageName, version) { blockResponse = { statusCode: 403, message: "Forbidden - blocked by safe-chain", @@ -80,13 +89,44 @@ function createRequestContext(targetUrl, eventEmitter) { }); } + /** @returns {RequestInterceptionHandler} */ + function build() { + /** @param {NodeJS.Dict | undefined} headers */ + function modifyRequestHeaders(headers) { + if (!headers) return; + + for (const func of reqheaderModificationFuncs) { + func(headers); + } + } + + /** + * @param {Buffer} body + * @returns {Buffer} + */ + function modifyBody(body) { + let modifiedBody = body; + + for (var func of modifyBodyFuncs) { + modifiedBody = func(body); + } + + return modifiedBody; + } + + return { + blockResponse, + modifyRequestHeaders: modifyRequestHeaders, + modifiesResponse: () => modifyBodyFuncs.length > 0, + modifyBody, + }; + } + return { targetUrl, - blockMalware, - build() { - return { - blockResponse, - }; - }, + blockMalware: blockMalwareSetup, + modifyRequestHeaders: (func) => reqheaderModificationFuncs.push(func), + modifyBody: (func) => modifyBodyFuncs.push(func), + build, }; } diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/modifyNpmInfo.js b/packages/safe-chain/src/registryProxy/interceptors/npm/modifyNpmInfo.js new file mode 100644 index 0000000..b69159a --- /dev/null +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/modifyNpmInfo.js @@ -0,0 +1,148 @@ +import { getMinimumPackageAgeHours } from "../../../config/settings.js"; +import { ui } from "../../../environment/userInteraction.js"; + +/** + * @param {NodeJS.Dict} headers + */ +export function modifyNpmInfoRequestHeaders(headers) { + if (headers["accept"]?.includes("application/vnd.npm.install-v1+json")) { + // The npm registry sometimes serves a more compact format that lacks + // the time metadata we need to filter out too new packages. + // Force the registry to return the full metadata by changing the Accept header. + headers["accept"] = "application/json"; + } +} + +/** + * @param {string} url + * @returns {boolean} + */ +export function isPackageInfoUrl(url) { + // Remove query string and fragment to get the actual path + const urlWithoutParams = url.split("?")[0].split("#")[0]; + + // Tarball downloads end with .tgz + if (urlWithoutParams.endsWith(".tgz")) return false; + + // Special endpoints start with /-/ and should not be modified + // Examples: /-/npm/v1/security/advisories/bulk, /-/v1/search, /-/package/foo/access + if (urlWithoutParams.includes("/-/")) return false; + + // Everything else is package metadata that can be modified + return true; +} +/** + * + * @param {Buffer} body + * @returns Buffer + */ +export function modifyNpmInfoResponse(body) { + try { + if (body.byteLength === 0) { + return body; + } + + const bodyContent = body.toString("utf8"); + const bodyJson = JSON.parse(bodyContent); + + if (!bodyJson.time || !bodyJson["dist-tags"] || !bodyJson.versions) { + // Just return the current body if the format is not + return body; + } + + const cutOff = new Date( + new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000 + ).toISOString(); + + const hasLatestTag = !!bodyJson["dist-tags"]["latest"]; + + const versions = Object.entries(bodyJson.time) + .map(([version, timestamp]) => ({ + version, + timestamp, + })) + .filter((x) => x.version !== "created" && x.version !== "modified"); + + for (const { version, timestamp } of versions) { + if (version === "created" || version === "modified") { + continue; + } + + if (timestamp > cutOff) { + deleteVersionFromJson(bodyJson, version); + continue; + } + } + + if (hasLatestTag && !bodyJson["dist-tags"]["latest"]) { + // The latest tag was removed because it contained a package younger than the treshold. + // A new latest tag needs to be calculated + bodyJson["dist-tags"]["latest"] = calculateLatestTag(bodyJson.time); + } + + return Buffer.from(JSON.stringify(bodyJson)); + } catch (err) { + ui.writeVerbose( + `Safe-chain: Package metadata not in expected format - bypassing modification. Error: ${err.message}` + ); + return body; + } +} + +/** + * @param {any} json + * @param {string} version + */ +function deleteVersionFromJson(json, version) { + ui.writeVerbose( + `Safe-chain: ${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).` + ); + + delete json.time[version]; + delete json.versions[version]; + + for (const [tag, distVersion] of Object.entries(json["dist-tags"])) { + if (version == distVersion) { + delete json["dist-tags"][tag]; + } + } +} + +/** + * @param {Record} tagList + * @returns {string | undefined} + */ +function calculateLatestTag(tagList) { + const entries = Object.entries(tagList).filter( + ([version, _]) => version !== "created" && version !== "modified" + ); + + const latestFullRelease = getMostRecentTag( + Object.fromEntries(entries.filter(([version, _]) => !version.includes("-"))) + ); + if (latestFullRelease) { + return latestFullRelease; + } + + const latestPrerelease = getMostRecentTag( + Object.fromEntries(entries.filter(([version, _]) => version.includes("-"))) + ); + return latestPrerelease; +} + +/** + * @param {Record} tagList + * @returns {string | undefined} + */ +function getMostRecentTag(tagList) { + let current, currentDate; + + for (const [version, timestamp] of Object.entries(tagList)) { + if (!currentDate || currentDate < timestamp) { + current = version; + currentDate = timestamp; + } + } + + return current; +} diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js index 0514636..467e5f0 100644 --- a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js @@ -1,7 +1,11 @@ -import { getMinimumPackageAgeHours } from "../../../config/settings.js"; import { isMalwarePackage } from "../../../scanning/audit/index.js"; import { interceptRequests } from "../interceptorBuilder.js"; -import { ui } from "../../../environment/userInteraction.js"; +import { + isPackageInfoUrl, + modifyNpmInfoRequestHeaders, + modifyNpmInfoResponse, +} from "./modifyNpmInfo.js"; +import { parseNpmPackageUrl } from "./parseNpmPackageUrl.js"; const knownJsRegistries = ["registry.npmjs.org", "registry.yarnpkg.com"]; @@ -29,197 +33,14 @@ function buildNpmInterceptor(registry) { reqContext.targetUrl, registry ); + if (await isMalwarePackage(packageName, version)) { reqContext.blockMalware(packageName, version); } if (isPackageInfoUrl(reqContext.targetUrl)) { - reqContext.modifyRequestHeaders((headers) => { - if ( - headers["accept"]?.includes("application/vnd.npm.install-v1+json") - ) { - // The npm registry sometimes serves a more compact format that lacks - // the time metadata we need to filter out too new packages. - // Force the registry to return the full metadata by changing the Accept header. - headers["accept"] = "application/json"; - } - }); - - reqContext.modifyResponse((res) => { - res.modifyBody(modifyNpmInfoRequestBody); - }); + reqContext.modifyRequestHeaders(modifyNpmInfoRequestHeaders); + reqContext.modifyBody(modifyNpmInfoResponse); } }); } - -/** - * @param {string} url - * @param {string} registry - * @returns {{packageName: string | undefined, version: string | undefined}} - */ -function parseNpmPackageUrl(url, registry) { - let packageName, version; - if (!registry || !url.endsWith(".tgz")) { - return { packageName, version }; - } - - const registryIndex = url.indexOf(registry); - const afterRegistry = url.substring(registryIndex + registry.length + 1); // +1 to skip the slash - - const separatorIndex = afterRegistry.indexOf("/-/"); - if (separatorIndex === -1) { - return { packageName, version }; - } - - packageName = afterRegistry.substring(0, separatorIndex); - const filename = afterRegistry.substring( - separatorIndex + 3, - afterRegistry.length - 4 - ); // Remove /-/ and .tgz - - // Extract version from filename - // For scoped packages like @babel/core, the filename is core-7.21.4.tgz - // For regular packages like lodash, the filename is lodash-4.17.21.tgz - if (packageName.startsWith("@")) { - const scopedPackageName = packageName.substring( - packageName.lastIndexOf("/") + 1 - ); - if (filename.startsWith(scopedPackageName + "-")) { - version = filename.substring(scopedPackageName.length + 1); - } - } else { - if (filename.startsWith(packageName + "-")) { - version = filename.substring(packageName.length + 1); - } - } - - return { packageName, version }; -} - -/** - * @param {string} url - * @returns {boolean} - */ -function isPackageInfoUrl(url) { - // Remove query string and fragment to get the actual path - const urlWithoutParams = url.split("?")[0].split("#")[0]; - - // Tarball downloads end with .tgz - if (urlWithoutParams.endsWith(".tgz")) return false; - - // Special endpoints start with /-/ and should not be modified - // Examples: /-/npm/v1/security/advisories/bulk, /-/v1/search, /-/package/foo/access - if (urlWithoutParams.includes("/-/")) return false; - - // Everything else is package metadata that can be modified - return true; -} - -/** - * - * @param {Buffer} body - * @returns Buffer - */ -function modifyNpmInfoRequestBody(body) { - try { - const bodyContent = body.toString("utf8"); - const bodyJson = JSON.parse(bodyContent); - - if (!bodyJson.time || !bodyJson["dist-tags"] || !bodyJson.versions) { - // Just return the body if the - return body; - } - - const cutOff = new Date( - new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000 - ).toISOString(); - - const hasLatestTag = !!bodyJson["dist-tags"]["latest"]; - - const versions = Object.entries(bodyJson.time) - .map(([version, timestamp]) => ({ - version, - timestamp, - })) - .filter((x) => x.version != "created" && x.version != "modified"); - - for (const { version, timestamp } of versions) { - if (version === "created" || version === "modified") { - continue; - } - - if (timestamp > cutOff) { - deleteVersionFromJson(bodyJson, version); - continue; - } - } - - if (hasLatestTag && !bodyJson["dist-tags"]["latest"]) { - // The latest tag was removed because it contained a package younger than the treshold. - // A new latest tag needs to be calculated - bodyJson["dist-tags"]["latest"] = calculateLatestTag(bodyJson); - } - - return Buffer.from(JSON.stringify(bodyJson)); - } catch (err) { - // TODO: better error handling - return body; - } -} - -function deleteVersionFromJson(json, version) { - ui.writeVerbose( - `Safe-chain: Deleting ${version} from npm info request, it's newer than the minimumPackageAgeInHours` - ); - - delete json.time[version]; - delete json.versions[version]; - - for (const [tag, distVersion] of Object.entries(json["dist-tags"])) { - if (version == distVersion) { - delete json["dist-tags"][tag]; - } - } -} - -function calculateLatestTag(json) { - if (!json.time) { - return undefined; - } - - let latest, preview, latestDate, previewDate; - - for (const [version, timestamp] of Object.entries(json.time)) { - if (version == "created" || version == "modified") continue; - - if (version.includes("-")) { - // preview versions include "-" in the name - [preview, previewDate] = getLatest( - preview, - previewDate, - version, - timestamp - ); - } else { - [latest, latestDate] = getLatest(latest, latestDate, version, timestamp); - } - } - - if (latest) { - return latest; - } else { - return preview; - } - - function getLatest(currentLatest, currentLatestDate, version, timestamp) { - if (!currentLatest) { - return [version, timestamp]; - } - - if (timestamp > currentLatestDate) { - return [version, timestamp]; - } - - return [currentLatest, currentLatestDate]; - } -} diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js index ea23f9e..269a241 100644 --- a/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js @@ -1,5 +1,6 @@ import { describe, it, mock } from "node:test"; import assert from "node:assert"; +import { buffer } from "node:stream/consumers"; describe("npmInterceptor minimum package age", async () => { let minimumPackageAgeSettings = 48; @@ -17,6 +18,19 @@ describe("npmInterceptor minimum package age", async () => { }, }, }); + mock.module("../../../environment/userInteraction.js", { + namedExports: { + ui: { + startProcess: () => {}, + writeError: () => {}, + writeInformation: () => {}, + writeWarning: () => {}, + writeVerbose: () => {}, + writeExitWithoutInstallingMaliciousPackages: () => {}, + emptyLine: () => {}, + }, + }, + }); const { npmInterceptorForUrl } = await import("./npmInterceptor.js"); for (const packageInfoUrl of [ @@ -128,6 +142,7 @@ describe("npmInterceptor minimum package age", async () => { }, time: { created: getDate(-365 * 24), + modified: getDate(-3), ["1.0.0"]: getDate(-7), // cutoff-date here ["2.0.0"]: getDate(-4), @@ -138,7 +153,7 @@ describe("npmInterceptor minimum package age", async () => { const modifiedJson = JSON.parse(modifiedBody); - assert.equal(Object.keys(modifiedJson.time).length, 2); + assert.equal(Object.keys(modifiedJson.time).length, 3); assert.equal(Object.keys(modifiedJson.versions).length, 1); assert.ok(Object.keys(modifiedJson.time).includes("1.0.0")); assert.ok(Object.keys(modifiedJson.versions).includes("1.0.0")); @@ -166,6 +181,7 @@ describe("npmInterceptor minimum package age", async () => { }, time: { created: getDate(-365 * 24), + modified: getDate(-3), ["1.0.0"]: getDate(-7), ["0.0.1"]: getDate(-8), // package order: this package is older than 1.0.0, it should not be considered latest ["2.0.0-alpha"]: getDate(-6), //package is a pre-release, it should not be latest @@ -200,6 +216,7 @@ describe("npmInterceptor minimum package age", async () => { }, time: { created: getDate(-365 * 24), + modified: getDate(-4), ["1.0.0"]: getDate(-7), // cutoff-date here ["2.0.0-alpha"]: getDate(-4), @@ -220,12 +237,14 @@ describe("npmInterceptor minimum package age", async () => { */ async function runModifyNpmInfoRequest(url, body) { const interceptor = npmInterceptorForUrl(url); - const requestInterceptor = await interceptor.handleRequest(url); - const responseInterceptor = requestInterceptor.handleResponse(); + const requestHandler = await interceptor.handleRequest(url); - const modifiedBuffer = responseInterceptor.modifyBody(Buffer.from(body)); + if (requestHandler.modifiesResponse()) { + const modifiedBuffer = requestHandler.modifyBody(Buffer.from(body)); + return modifiedBuffer.toString("utf8"); + } - return modifiedBuffer.toString("utf8"); + return buffer; } function getDate(plusHours) { diff --git a/packages/safe-chain/src/registryProxy/interceptors/npm/parseNpmPackageUrl.js b/packages/safe-chain/src/registryProxy/interceptors/npm/parseNpmPackageUrl.js new file mode 100644 index 0000000..fa256d4 --- /dev/null +++ b/packages/safe-chain/src/registryProxy/interceptors/npm/parseNpmPackageUrl.js @@ -0,0 +1,43 @@ +/** + * @param {string} url + * @param {string} registry + * @returns {{packageName: string | undefined, version: string | undefined}} + */ +export function parseNpmPackageUrl(url, registry) { + let packageName, version; + if (!registry || !url.endsWith(".tgz")) { + return { packageName, version }; + } + + const registryIndex = url.indexOf(registry); + const afterRegistry = url.substring(registryIndex + registry.length + 1); // +1 to skip the slash + + const separatorIndex = afterRegistry.indexOf("/-/"); + if (separatorIndex === -1) { + return { packageName, version }; + } + + packageName = afterRegistry.substring(0, separatorIndex); + const filename = afterRegistry.substring( + separatorIndex + 3, + afterRegistry.length - 4 + ); // Remove /-/ and .tgz + + // Extract version from filename + // For scoped packages like @babel/core, the filename is core-7.21.4.tgz + // For regular packages like lodash, the filename is lodash-4.17.21.tgz + if (packageName.startsWith("@")) { + const scopedPackageName = packageName.substring( + packageName.lastIndexOf("/") + 1 + ); + if (filename.startsWith(scopedPackageName + "-")) { + version = filename.substring(scopedPackageName.length + 1); + } + } else { + if (filename.startsWith(packageName + "-")) { + version = filename.substring(packageName.length + 1); + } + } + + return { packageName, version }; +} diff --git a/packages/safe-chain/src/registryProxy/mitmRequestHandler.js b/packages/safe-chain/src/registryProxy/mitmRequestHandler.js index aa1391e..70e0a51 100644 --- a/packages/safe-chain/src/registryProxy/mitmRequestHandler.js +++ b/packages/safe-chain/src/registryProxy/mitmRequestHandler.js @@ -6,7 +6,6 @@ import { gunzipSync, gzipSync } from "zlib"; /** * @typedef {import("./interceptors/interceptorBuilder.js").Interceptor} Interceptor - * @typedef {import("./interceptors/requestInterceptorBuilder.js").RequestInterceptor} RequestInterceptor */ /** @@ -113,10 +112,10 @@ function getRequestPathAndQuery(url) { * @param {import("http").IncomingMessage} req * @param {string} hostname * @param {import("http").ServerResponse} res - * @param {RequestInterceptor} requestInterceptor + * @param {import("./interceptors/interceptorBuilder.js").RequestInterceptionHandler} requestHandler */ -function forwardRequest(req, hostname, res, requestInterceptor) { - const proxyReq = createProxyRequest(hostname, req, res, requestInterceptor); +function forwardRequest(req, hostname, res, requestHandler) { + const proxyReq = createProxyRequest(hostname, req, res, requestHandler); proxyReq.on("error", (err) => { ui.writeVerbose( @@ -147,16 +146,16 @@ function forwardRequest(req, hostname, res, requestInterceptor) { * @param {string} hostname * @param {import("http").IncomingMessage} req * @param {import("http").ServerResponse} res - * @param {RequestInterceptor} requestInterceptor + * @param {import("./interceptors/interceptorBuilder.js").RequestInterceptionHandler} requestHandler * * @returns {import("http").ClientRequest} */ -function createProxyRequest(hostname, req, res, requestInterceptor) { +function createProxyRequest(hostname, req, res, requestHandler) { const headers = { ...req.headers }; if (headers.host) { delete headers.host; } - requestInterceptor.modifyRequestHeaders(headers); + requestHandler.modifyRequestHeaders(headers); /** @type {import("http").RequestOptions} */ const options = { @@ -191,9 +190,7 @@ function createProxyRequest(hostname, req, res, requestInterceptor) { } res.writeHead(proxyRes.statusCode, proxyRes.headers); - if (requestInterceptor.modifiesResponse()) { - const responseInterceptor = requestInterceptor.handleResponse(); - + if (requestHandler.modifiesResponse()) { /** @type {Array} */ let chunks = []; @@ -207,7 +204,7 @@ function createProxyRequest(hostname, req, res, requestInterceptor) { buffer = gunzipSync(buffer); } - buffer = responseInterceptor.modifyBody(buffer); + buffer = requestHandler.modifyBody(buffer); if (proxyRes.headers["content-encoding"] === "gzip") { buffer = gzipSync(buffer);