Finish npm info modification.

packages/safe-chain/src/config/settings.js

@@ -37,6 +37,7 @@ export function setEcoSystem(setting) {
   ecosystemSettings.ecoSystem = setting;
 }
 
+const defaultMinimumPackageAge = 24;
 export function getMinimumPackageAgeHours() {
-  return 24 * 6;
+  return defaultMinimumPackageAge;
 }

packages/safe-chain/src/registryProxy/interceptors/interceptorBuilder.js

@@ -10,11 +10,16 @@ import { EventEmitter } from "events";
  * @typedef {Object} RequestInterceptionContext
  * @property {string} targetUrl
  * @property {(packageName: string | undefined, version: string | undefined) => void} blockMalware
+ * @property {(modificationFunc: (headers: NodeJS.Dict<string | string[]>) => void) => void} modifyRequestHeaders
+ * @property {(modificationFunc: (body: Buffer) => Buffer) => void} modifyBody
  * @property {() => RequestInterceptionHandler} build
  *
  *
  * @typedef {Object} RequestInterceptionHandler
  * @property {{statusCode: number, message: string} | undefined} blockResponse
+ * @property {(headers: NodeJS.Dict<string | string[]> | undefined) => void} modifyRequestHeaders
+ * @property {() => boolean} modifiesResponse
+ * @property {(body: Buffer) => Buffer} modifyBody
  */
 
 /**
@@ -60,12 +65,16 @@ function buildInterceptor(requestHandlers) {
 function createRequestContext(targetUrl, eventEmitter) {
   /** @type {{statusCode: number, message: string} | undefined}  */
   let blockResponse = undefined;
+  /** @type {Array<(headers: NodeJS.Dict<string | string[]>) => void>} */
+  let reqheaderModificationFuncs = [];
+  /** @type {Array<(body: Buffer) => Buffer>} */
+  let modifyBodyFuncs = [];
 
   /**
    * @param {string | undefined} packageName
    * @param {string | undefined} version
    */
-  function blockMalware(packageName, version) {
+  function blockMalwareSetup(packageName, version) {
     blockResponse = {
       statusCode: 403,
       message: "Forbidden - blocked by safe-chain",
@@ -80,13 +89,44 @@ function createRequestContext(targetUrl, eventEmitter) {
     });
   }
 
-  return {
+  /** @returns {RequestInterceptionHandler} */
-    targetUrl,
+  function build() {
-    blockMalware,
+    /** @param {NodeJS.Dict<string | string[]> | undefined} headers */
-    build() {
+    function modifyRequestHeaders(headers) {
+      if (!headers) return;
+
+      for (const func of reqheaderModificationFuncs) {
+        func(headers);
+      }
+    }
+
+    /**
+     * @param {Buffer} body
+     * @returns {Buffer}
+     */
+    function modifyBody(body) {
+      let modifiedBody = body;
+
+      for (var func of modifyBodyFuncs) {
+        modifiedBody = func(body);
+      }
+
+      return modifiedBody;
+    }
+
     return {
       blockResponse,
+      modifyRequestHeaders: modifyRequestHeaders,
+      modifiesResponse: () => modifyBodyFuncs.length > 0,
+      modifyBody,
     };
-    },
+  }
+
+  return {
+    targetUrl,
+    blockMalware: blockMalwareSetup,
+    modifyRequestHeaders: (func) => reqheaderModificationFuncs.push(func),
+    modifyBody: (func) => modifyBodyFuncs.push(func),
+    build,
   };
 }

packages/safe-chain/src/registryProxy/interceptors/npm/modifyNpmInfo.js

@@ -0,0 +1,148 @@
+import { getMinimumPackageAgeHours } from "../../../config/settings.js";
+import { ui } from "../../../environment/userInteraction.js";
+
+/**
+ * @param {NodeJS.Dict<string | string[]>} headers
+ */
+export function modifyNpmInfoRequestHeaders(headers) {
+  if (headers["accept"]?.includes("application/vnd.npm.install-v1+json")) {
+    // The npm registry sometimes serves a more compact format that lacks
+    // the time metadata we need to filter out too new packages.
+    // Force the registry to return the full metadata by changing the Accept header.
+    headers["accept"] = "application/json";
+  }
+}
+
+/**
+ * @param {string} url
+ * @returns {boolean}
+ */
+export function isPackageInfoUrl(url) {
+  // Remove query string and fragment to get the actual path
+  const urlWithoutParams = url.split("?")[0].split("#")[0];
+
+  // Tarball downloads end with .tgz
+  if (urlWithoutParams.endsWith(".tgz")) return false;
+
+  // Special endpoints start with /-/ and should not be modified
+  // Examples: /-/npm/v1/security/advisories/bulk, /-/v1/search, /-/package/foo/access
+  if (urlWithoutParams.includes("/-/")) return false;
+
+  // Everything else is package metadata that can be modified
+  return true;
+}
+/**
+ *
+ * @param {Buffer} body
+ * @returns Buffer
+ */
+export function modifyNpmInfoResponse(body) {
+  try {
+    if (body.byteLength === 0) {
+      return body;
+    }
+
+    const bodyContent = body.toString("utf8");
+    const bodyJson = JSON.parse(bodyContent);
+
+    if (!bodyJson.time || !bodyJson["dist-tags"] || !bodyJson.versions) {
+      // Just return the current body if the format is not
+      return body;
+    }
+
+    const cutOff = new Date(
+      new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
+    ).toISOString();
+
+    const hasLatestTag = !!bodyJson["dist-tags"]["latest"];
+
+    const versions = Object.entries(bodyJson.time)
+      .map(([version, timestamp]) => ({
+        version,
+        timestamp,
+      }))
+      .filter((x) => x.version !== "created" && x.version !== "modified");
+
+    for (const { version, timestamp } of versions) {
+      if (version === "created" || version === "modified") {
+        continue;
+      }
+
+      if (timestamp > cutOff) {
+        deleteVersionFromJson(bodyJson, version);
+        continue;
+      }
+    }
+
+    if (hasLatestTag && !bodyJson["dist-tags"]["latest"]) {
+      // The latest tag was removed because it contained a package younger than the treshold.
+      // A new latest tag needs to be calculated
+      bodyJson["dist-tags"]["latest"] = calculateLatestTag(bodyJson.time);
+    }
+
+    return Buffer.from(JSON.stringify(bodyJson));
+  } catch (err) {
+    ui.writeVerbose(
+      `Safe-chain: Package metadata not in expected format - bypassing modification. Error: ${err.message}`
+    );
+    return body;
+  }
+}
+
+/**
+ * @param {any} json
+ * @param {string} version
+ */
+function deleteVersionFromJson(json, version) {
+  ui.writeVerbose(
+    `Safe-chain: ${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`
+  );
+
+  delete json.time[version];
+  delete json.versions[version];
+
+  for (const [tag, distVersion] of Object.entries(json["dist-tags"])) {
+    if (version == distVersion) {
+      delete json["dist-tags"][tag];
+    }
+  }
+}
+
+/**
+ * @param {Record<string, string>} tagList
+ * @returns {string | undefined}
+ */
+function calculateLatestTag(tagList) {
+  const entries = Object.entries(tagList).filter(
+    ([version, _]) => version !== "created" && version !== "modified"
+  );
+
+  const latestFullRelease = getMostRecentTag(
+    Object.fromEntries(entries.filter(([version, _]) => !version.includes("-")))
+  );
+  if (latestFullRelease) {
+    return latestFullRelease;
+  }
+
+  const latestPrerelease = getMostRecentTag(
+    Object.fromEntries(entries.filter(([version, _]) => version.includes("-")))
+  );
+  return latestPrerelease;
+}
+
+/**
+ * @param {Record<string, string>} tagList
+ * @returns {string | undefined}
+ */
+function getMostRecentTag(tagList) {
+  let current, currentDate;
+
+  for (const [version, timestamp] of Object.entries(tagList)) {
+    if (!currentDate || currentDate < timestamp) {
+      current = version;
+      currentDate = timestamp;
+    }
+  }
+
+  return current;
+}

packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.js

@@ -1,7 +1,11 @@
-import { getMinimumPackageAgeHours } from "../../../config/settings.js";
 import { isMalwarePackage } from "../../../scanning/audit/index.js";
 import { interceptRequests } from "../interceptorBuilder.js";
-import { ui } from "../../../environment/userInteraction.js";
+import {
+  isPackageInfoUrl,
+  modifyNpmInfoRequestHeaders,
+  modifyNpmInfoResponse,
+} from "./modifyNpmInfo.js";
+import { parseNpmPackageUrl } from "./parseNpmPackageUrl.js";
 
 const knownJsRegistries = ["registry.npmjs.org", "registry.yarnpkg.com"];
 
@@ -29,197 +33,14 @@ function buildNpmInterceptor(registry) {
       reqContext.targetUrl,
       registry
     );
+
     if (await isMalwarePackage(packageName, version)) {
       reqContext.blockMalware(packageName, version);
     }
 
     if (isPackageInfoUrl(reqContext.targetUrl)) {
-      reqContext.modifyRequestHeaders((headers) => {
+      reqContext.modifyRequestHeaders(modifyNpmInfoRequestHeaders);
-        if (
+      reqContext.modifyBody(modifyNpmInfoResponse);
-          headers["accept"]?.includes("application/vnd.npm.install-v1+json")
-        ) {
-          // The npm registry sometimes serves a more compact format that lacks
-          // the time metadata we need to filter out too new packages.
-          // Force the registry to return the full metadata by changing the Accept header.
-          headers["accept"] = "application/json";
-        }
-      });
-
-      reqContext.modifyResponse((res) => {
-        res.modifyBody(modifyNpmInfoRequestBody);
-      });
     }
   });
 }
-
-/**
- * @param {string} url
- * @param {string} registry
- * @returns {{packageName: string | undefined, version: string | undefined}}
- */
-function parseNpmPackageUrl(url, registry) {
-  let packageName, version;
-  if (!registry || !url.endsWith(".tgz")) {
-    return { packageName, version };
-  }
-
-  const registryIndex = url.indexOf(registry);
-  const afterRegistry = url.substring(registryIndex + registry.length + 1); // +1 to skip the slash
-
-  const separatorIndex = afterRegistry.indexOf("/-/");
-  if (separatorIndex === -1) {
-    return { packageName, version };
-  }
-
-  packageName = afterRegistry.substring(0, separatorIndex);
-  const filename = afterRegistry.substring(
-    separatorIndex + 3,
-    afterRegistry.length - 4
-  ); // Remove /-/ and .tgz
-
-  // Extract version from filename
-  // For scoped packages like @babel/core, the filename is core-7.21.4.tgz
-  // For regular packages like lodash, the filename is lodash-4.17.21.tgz
-  if (packageName.startsWith("@")) {
-    const scopedPackageName = packageName.substring(
-      packageName.lastIndexOf("/") + 1
-    );
-    if (filename.startsWith(scopedPackageName + "-")) {
-      version = filename.substring(scopedPackageName.length + 1);
-    }
-  } else {
-    if (filename.startsWith(packageName + "-")) {
-      version = filename.substring(packageName.length + 1);
-    }
-  }
-
-  return { packageName, version };
-}
-
-/**
- * @param {string} url
- * @returns {boolean}
- */
-function isPackageInfoUrl(url) {
-  // Remove query string and fragment to get the actual path
-  const urlWithoutParams = url.split("?")[0].split("#")[0];
-
-  // Tarball downloads end with .tgz
-  if (urlWithoutParams.endsWith(".tgz")) return false;
-
-  // Special endpoints start with /-/ and should not be modified
-  // Examples: /-/npm/v1/security/advisories/bulk, /-/v1/search, /-/package/foo/access
-  if (urlWithoutParams.includes("/-/")) return false;
-
-  // Everything else is package metadata that can be modified
-  return true;
-}
-
-/**
- *
- * @param {Buffer} body
- * @returns Buffer
- */
-function modifyNpmInfoRequestBody(body) {
-  try {
-    const bodyContent = body.toString("utf8");
-    const bodyJson = JSON.parse(bodyContent);
-
-    if (!bodyJson.time || !bodyJson["dist-tags"] || !bodyJson.versions) {
-      // Just return the body if the
-      return body;
-    }
-
-    const cutOff = new Date(
-      new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
-    ).toISOString();
-
-    const hasLatestTag = !!bodyJson["dist-tags"]["latest"];
-
-    const versions = Object.entries(bodyJson.time)
-      .map(([version, timestamp]) => ({
-        version,
-        timestamp,
-      }))
-      .filter((x) => x.version != "created" && x.version != "modified");
-
-    for (const { version, timestamp } of versions) {
-      if (version === "created" || version === "modified") {
-        continue;
-      }
-
-      if (timestamp > cutOff) {
-        deleteVersionFromJson(bodyJson, version);
-        continue;
-      }
-    }
-
-    if (hasLatestTag && !bodyJson["dist-tags"]["latest"]) {
-      // The latest tag was removed because it contained a package younger than the treshold.
-      // A new latest tag needs to be calculated
-      bodyJson["dist-tags"]["latest"] = calculateLatestTag(bodyJson);
-    }
-
-    return Buffer.from(JSON.stringify(bodyJson));
-  } catch (err) {
-    // TODO: better error handling
-    return body;
-  }
-}
-
-function deleteVersionFromJson(json, version) {
-  ui.writeVerbose(
-    `Safe-chain: Deleting ${version} from npm info request, it's newer than the minimumPackageAgeInHours`
-  );
-
-  delete json.time[version];
-  delete json.versions[version];
-
-  for (const [tag, distVersion] of Object.entries(json["dist-tags"])) {
-    if (version == distVersion) {
-      delete json["dist-tags"][tag];
-    }
-  }
-}
-
-function calculateLatestTag(json) {
-  if (!json.time) {
-    return undefined;
-  }
-
-  let latest, preview, latestDate, previewDate;
-
-  for (const [version, timestamp] of Object.entries(json.time)) {
-    if (version == "created" || version == "modified") continue;
-
-    if (version.includes("-")) {
-      // preview versions include "-" in the name
-      [preview, previewDate] = getLatest(
-        preview,
-        previewDate,
-        version,
-        timestamp
-      );
-    } else {
-      [latest, latestDate] = getLatest(latest, latestDate, version, timestamp);
-    }
-  }
-
-  if (latest) {
-    return latest;
-  } else {
-    return preview;
-  }
-
-  function getLatest(currentLatest, currentLatestDate, version, timestamp) {
-    if (!currentLatest) {
-      return [version, timestamp];
-    }
-
-    if (timestamp > currentLatestDate) {
-      return [version, timestamp];
-    }
-
-    return [currentLatest, currentLatestDate];
-  }
-}

packages/safe-chain/src/registryProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js

@@ -1,5 +1,6 @@
 import { describe, it, mock } from "node:test";
 import assert from "node:assert";
+import { buffer } from "node:stream/consumers";
 
 describe("npmInterceptor minimum package age", async () => {
   let minimumPackageAgeSettings = 48;
@@ -17,6 +18,19 @@ describe("npmInterceptor minimum package age", async () => {
       },
     },
   });
+  mock.module("../../../environment/userInteraction.js", {
+    namedExports: {
+      ui: {
+        startProcess: () => {},
+        writeError: () => {},
+        writeInformation: () => {},
+        writeWarning: () => {},
+        writeVerbose: () => {},
+        writeExitWithoutInstallingMaliciousPackages: () => {},
+        emptyLine: () => {},
+      },
+    },
+  });
   const { npmInterceptorForUrl } = await import("./npmInterceptor.js");
 
   for (const packageInfoUrl of [
@@ -128,6 +142,7 @@ describe("npmInterceptor minimum package age", async () => {
         },
         time: {
           created: getDate(-365 * 24),
+          modified: getDate(-3),
           ["1.0.0"]: getDate(-7),
           // cutoff-date here
           ["2.0.0"]: getDate(-4),
@@ -138,7 +153,7 @@ describe("npmInterceptor minimum package age", async () => {
 
     const modifiedJson = JSON.parse(modifiedBody);
 
-    assert.equal(Object.keys(modifiedJson.time).length, 2);
+    assert.equal(Object.keys(modifiedJson.time).length, 3);
     assert.equal(Object.keys(modifiedJson.versions).length, 1);
     assert.ok(Object.keys(modifiedJson.time).includes("1.0.0"));
     assert.ok(Object.keys(modifiedJson.versions).includes("1.0.0"));
@@ -166,6 +181,7 @@ describe("npmInterceptor minimum package age", async () => {
         },
         time: {
           created: getDate(-365 * 24),
+          modified: getDate(-3),
           ["1.0.0"]: getDate(-7),
           ["0.0.1"]: getDate(-8), // package order: this package is older than 1.0.0, it should not be considered latest
           ["2.0.0-alpha"]: getDate(-6), //package is a pre-release, it should not be latest
@@ -200,6 +216,7 @@ describe("npmInterceptor minimum package age", async () => {
         },
         time: {
           created: getDate(-365 * 24),
+          modified: getDate(-4),
           ["1.0.0"]: getDate(-7),
           // cutoff-date here
           ["2.0.0-alpha"]: getDate(-4),
@@ -220,14 +237,16 @@ describe("npmInterceptor minimum package age", async () => {
    */
   async function runModifyNpmInfoRequest(url, body) {
     const interceptor = npmInterceptorForUrl(url);
-    const requestInterceptor = await interceptor.handleRequest(url);
+    const requestHandler = await interceptor.handleRequest(url);
-    const responseInterceptor = requestInterceptor.handleResponse();
-
-    const modifiedBuffer = responseInterceptor.modifyBody(Buffer.from(body));
 
+    if (requestHandler.modifiesResponse()) {
+      const modifiedBuffer = requestHandler.modifyBody(Buffer.from(body));
       return modifiedBuffer.toString("utf8");
     }
 
+    return buffer;
+  }
+
   function getDate(plusHours) {
     const date = new Date();
     date.setHours(date.getHours() + plusHours);

packages/safe-chain/src/registryProxy/interceptors/npm/parseNpmPackageUrl.js

@@ -0,0 +1,43 @@
+/**
+ * @param {string} url
+ * @param {string} registry
+ * @returns {{packageName: string | undefined, version: string | undefined}}
+ */
+export function parseNpmPackageUrl(url, registry) {
+  let packageName, version;
+  if (!registry || !url.endsWith(".tgz")) {
+    return { packageName, version };
+  }
+
+  const registryIndex = url.indexOf(registry);
+  const afterRegistry = url.substring(registryIndex + registry.length + 1); // +1 to skip the slash
+
+  const separatorIndex = afterRegistry.indexOf("/-/");
+  if (separatorIndex === -1) {
+    return { packageName, version };
+  }
+
+  packageName = afterRegistry.substring(0, separatorIndex);
+  const filename = afterRegistry.substring(
+    separatorIndex + 3,
+    afterRegistry.length - 4
+  ); // Remove /-/ and .tgz
+
+  // Extract version from filename
+  // For scoped packages like @babel/core, the filename is core-7.21.4.tgz
+  // For regular packages like lodash, the filename is lodash-4.17.21.tgz
+  if (packageName.startsWith("@")) {
+    const scopedPackageName = packageName.substring(
+      packageName.lastIndexOf("/") + 1
+    );
+    if (filename.startsWith(scopedPackageName + "-")) {
+      version = filename.substring(scopedPackageName.length + 1);
+    }
+  } else {
+    if (filename.startsWith(packageName + "-")) {
+      version = filename.substring(packageName.length + 1);
+    }
+  }
+
+  return { packageName, version };
+}

packages/safe-chain/src/registryProxy/mitmRequestHandler.js

@@ -6,7 +6,6 @@ import { gunzipSync, gzipSync } from "zlib";
 
 /**
  * @typedef {import("./interceptors/interceptorBuilder.js").Interceptor} Interceptor
- * @typedef {import("./interceptors/requestInterceptorBuilder.js").RequestInterceptor} RequestInterceptor
  */
 
 /**
@@ -113,10 +112,10 @@ function getRequestPathAndQuery(url) {
  * @param {import("http").IncomingMessage} req
  * @param {string} hostname
  * @param {import("http").ServerResponse} res
- * @param {RequestInterceptor} requestInterceptor
+ * @param {import("./interceptors/interceptorBuilder.js").RequestInterceptionHandler} requestHandler
  */
-function forwardRequest(req, hostname, res, requestInterceptor) {
+function forwardRequest(req, hostname, res, requestHandler) {
-  const proxyReq = createProxyRequest(hostname, req, res, requestInterceptor);
+  const proxyReq = createProxyRequest(hostname, req, res, requestHandler);
 
   proxyReq.on("error", (err) => {
     ui.writeVerbose(
@@ -147,16 +146,16 @@ function forwardRequest(req, hostname, res, requestInterceptor) {
  * @param {string} hostname
  * @param {import("http").IncomingMessage} req
  * @param {import("http").ServerResponse} res
- * @param {RequestInterceptor} requestInterceptor
+ * @param {import("./interceptors/interceptorBuilder.js").RequestInterceptionHandler} requestHandler
  *
  * @returns {import("http").ClientRequest}
  */
-function createProxyRequest(hostname, req, res, requestInterceptor) {
+function createProxyRequest(hostname, req, res, requestHandler) {
   const headers = { ...req.headers };
   if (headers.host) {
     delete headers.host;
   }
-  requestInterceptor.modifyRequestHeaders(headers);
+  requestHandler.modifyRequestHeaders(headers);
 
   /** @type {import("http").RequestOptions} */
   const options = {
@@ -191,9 +190,7 @@ function createProxyRequest(hostname, req, res, requestInterceptor) {
     }
     res.writeHead(proxyRes.statusCode, proxyRes.headers);
 
-    if (requestInterceptor.modifiesResponse()) {
+    if (requestHandler.modifiesResponse()) {
-      const responseInterceptor = requestInterceptor.handleResponse();
-
       /** @type {Array<any>} */
       let chunks = [];
 
@@ -207,7 +204,7 @@ function createProxyRequest(hostname, req, res, requestInterceptor) {
           buffer = gunzipSync(buffer);
         }
 
-        buffer = responseInterceptor.modifyBody(buffer);
+        buffer = requestHandler.modifyBody(buffer);
 
         if (proxyRes.headers["content-encoding"] === "gzip") {
           buffer = gzipSync(buffer);