milliways-infosec/AikidoSec-safe-chain
7
0
Fork 0
mirror of https://github.com/AikidoSec/safe-chain.git synced 2026-05-26 12:10:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into rama-integration-beta

Browse source
This commit is contained in:
Sander Declerck 2026-05-04 12:40:20 +02:00
commit 64a825f43a
No known key found for this signature in database
130 changed files with 6144 additions and 2494 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/safe-chain/src/registryProxy/builtInProxy/certUtils.js
View file

@ -1,9 +1,8 @@
import forge from "node-forge";
import path from "path";
import fs from "fs";
import os from "os";
import { getCertsDir } from "../../config/safeChainDir.js";
const certFolder = path.join(os.homedir(), ".safe-chain", "certs");
const ca = loadCa();
const certCache = new Map();
@ -20,7 +19,7 @@ function createKeyIdentifier(publicKey) {
}
export function getCaCertPath() {
return path.join(certFolder, "ca-cert.pem");
return path.join(getCertsDir(), "ca-cert.pem");
}
/**
@ -112,6 +111,7 @@ export function generateCertForHost(hostname) {
}
function loadCa() {
const certFolder = getCertsDir();
const keyPath = path.join(certFolder, "ca-key.pem");
const certPath = path.join(certFolder, "ca-cert.pem");

71
packages/safe-chain/src/registryProxy/builtInProxy/certUtils.spec.js Normal file
View file

@ -0,0 +1,71 @@
import { describe, it, beforeEach, afterEach, mock } from "node:test";
import assert from "node:assert";
describe("certUtils", () => {
let installedSafeChainDir;
beforeEach(() => {
installedSafeChainDir = undefined;
mock.module("../../config/safeChainDir.js", {
namedExports: {
getSafeChainBaseDir: () => installedSafeChainDir ?? "/home/test/.safe-chain",
getCertsDir: () => `${installedSafeChainDir ?? "/home/test/.safe-chain"}/certs`,
},
});
});
afterEach(() => {
mock.reset();
});
it("stores CA certificates in the packaged install dir when available", async () => {
installedSafeChainDir = "/custom/safe-chain";
mock.module("fs", {
defaultExport: {
existsSync: () => false,
mkdirSync: () => {},
writeFileSync: () => {},
},
});
mock.module("node-forge", {
defaultExport: {
pki: {
getPublicKeyFingerprint: () => "fingerprint",
rsa: {
generateKeyPair: () => ({
publicKey: "public-key",
privateKey: "private-key",
}),
},
createCertificate: () => ({
publicKey: null,
serialNumber: "",
validity: {
notBefore: new Date(),
notAfter: new Date(),
},
setSubject: () => {},
setIssuer: () => {},
setExtensions: () => {},
sign: () => {},
}),
privateKeyToPem: () => "private-key-pem",
certificateToPem: () => "certificate-pem",
},
md: {
sha1: { create: () => "sha1" },
sha256: { create: () => "sha256" },
},
},
});
const { getCaCertPath } = await import("./certUtils.js");
assert.strictEqual(
getCaCertPath(),
"/custom/safe-chain/certs/ca-cert.pem",
);
});
});

23
packages/safe-chain/src/registryProxy/builtInProxy/createBuiltInProxyServer.js
View file

@ -4,10 +4,11 @@ import { mitmConnect } from "./mitmRequestHandler.js";
import { handleHttpProxyRequest } from "./plainHttpProxy.js";
import { ui } from "../../environment/userInteraction.js";
import { createInterceptorForUrl } from "./interceptors/createInterceptorForEcoSystem.js";
import { getHasSuppressedVersions } from "./interceptors/npm/modifyNpmInfo.js";
import { getCaCertPath } from "./certUtils.js";
import { readFileSync } from "fs";
import EventEmitter from "events";
import { cleanupCertBundle } from "../certBundle.js";
import { getHasSuppressedVersions } from "./interceptors/suppressedVersionsState.js";
/** *
* @returns {import("../registryProxy.js").SafeChainProxy} */
@ -73,12 +74,16 @@ export function createBuiltInProxyServer() {
return new Promise((resolve) => {
try {
server.close(() => {
cleanupCertBundle();
resolve();
});
} catch {
resolve();
}
setTimeout(() => resolve(), SERVER_STOP_TIMEOUT_MS);
setTimeout(() => {
cleanupCertBundle();
resolve();
}, SERVER_STOP_TIMEOUT_MS);
});
}
@ -104,7 +109,19 @@ export function createBuiltInProxyServer() {
) => {
emitter.emit("malwareBlocked", {
packageName: event.packageName,
packageVersion: event.version
packageVersion: event.version,
});
},
);
interceptor.on(
"minimumAgeRequestBlocked",
(
/** @type {import("./interceptors/interceptorBuilder.js").MinimumAgeRequestBlockedEvent} */ event,
) => {
emitter.emit("minimumAgeRequestBlocked", {
packageName: event.packageName,
packageVersion: event.version,
});
},
);

63
packages/safe-chain/src/registryProxy/builtInProxy/http-utils.js
View file

@ -15,3 +15,66 @@ export function getHeaderValueAsString(headers, headerName) {
return header;
}
/**
* Returns a copy of headers without the provided header names, matched
* either exactly or case-insensitively.
*
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @param {string[]} headerNames
* @param {{ caseInsensitive?: boolean }} [options]
* @returns {NodeJS.Dict<string | string[]> | undefined}
*/
export function omitHeaders(headers, headerNames, options = {}) {
if (!headers) {
return headers;
}
const omittedHeaderNames = new Set(
options.caseInsensitive
? headerNames.map((name) => name.toLowerCase())
: headerNames
);
/** @type {NodeJS.Dict<string | string[]>} */
const filteredHeaders = {};
for (const [headerName, value] of Object.entries(headers)) {
const comparableHeaderName = options.caseInsensitive
? headerName.toLowerCase()
: headerName;
if (!omittedHeaderNames.has(comparableHeaderName)) {
filteredHeaders[headerName] = value;
}
}
return filteredHeaders;
}
/**
* Remove headers that become stale when the response body is modified.
*
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @returns {void}
*/
export function clearCachingHeaders(headers) {
if (!headers) {
return;
}
const filteredHeaders = omitHeaders(headers, [
"etag",
"last-modified",
"cache-control",
"content-length",
]);
if (!filteredHeaders) {
return;
}
for (const key of Object.keys(headers)) {
delete headers[key];
}
Object.assign(headers, filteredHeaders);
}

2
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/createInterceptorForEcoSystem.js
View file

@ -4,7 +4,7 @@ import {
getEcoSystem,
} from "../../../config/settings.js";
import { npmInterceptorForUrl } from "./npm/npmInterceptor.js";
import { pipInterceptorForUrl } from "./pipInterceptor.js";
import { pipInterceptorForUrl } from "./pip/pipInterceptor.js";
/**
* @param {string} url

41
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/interceptorBuilder.js
View file

@ -10,6 +10,7 @@ import { EventEmitter } from "events";
* @typedef {Object} RequestInterceptionContext
* @property {string} targetUrl
* @property {(packageName: string | undefined, version: string | undefined) => void} blockMalware
* @property {(packageName: string, version: string, message: string) => void} blockMinimumAgeRequest
* @property {(modificationFunc: (headers: NodeJS.Dict<string | string[]>) => NodeJS.Dict<string | string[]>) => void} modifyRequestHeaders
* @property {(modificationFunc: (body: Buffer, headers: NodeJS.Dict<string | string[]> | undefined) => Buffer) => void} modifyBody
* @property {() => RequestInterceptionHandler} build
@ -26,6 +27,12 @@ import { EventEmitter } from "events";
* @property {string} version
* @property {string} targetUrl
* @property {number} timestamp
*
* @typedef {Object} MinimumAgeRequestBlockedEvent
* @property {string} packageName
* @property {string} version
* @property {string} targetUrl
* @property {number} timestamp
*/
/**
@ -81,10 +88,7 @@ function createRequestContext(targetUrl, eventEmitter) {
* @param {string | undefined} version
*/
function blockMalwareSetup(packageName, version) {
blockResponse = {
statusCode: 403,
message: "Forbidden - blocked by safe-chain",
};
blockResponse = createBlockResponse("Forbidden - blocked by safe-chain");
// Emit the malwareBlocked event
eventEmitter.emit("malwareBlocked", {
@ -95,6 +99,34 @@ function createRequestContext(targetUrl, eventEmitter) {
});
}
/**
* @param {string} message
*/
function blockMinimumAgeRequestSetup(
/** @type {string} */ packageName,
/** @type {string} */ version,
/** @type {string} */ message
) {
blockResponse = createBlockResponse(message);
eventEmitter.emit("minimumAgeRequestBlocked", {
packageName,
version,
targetUrl,
timestamp: Date.now(),
});
}
/**
* @param {string} message
* @returns {{statusCode: number, message: string}}
*/
function createBlockResponse(message) {
return {
statusCode: 403,
message,
};
}
/** @returns {RequestInterceptionHandler} */
function build() {
/**
@ -139,6 +171,7 @@ function createRequestContext(targetUrl, eventEmitter) {
return {
targetUrl,
blockMalware: blockMalwareSetup,
blockMinimumAgeRequest: blockMinimumAgeRequestSetup,
modifyRequestHeaders: (func) => reqheaderModificationFuncs.push(func),
modifyBody: (func) => modifyBodyFuncs.push(func),
build,

33
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/minimumPackageAgeExclusions.js Normal file
View file

@ -0,0 +1,33 @@
import { getMinimumPackageAgeExclusions, getEcoSystem } from "../../../config/settings.js";
import { getEquivalentPackageNames } from "../../../scanning/packageNameVariants.js";
/**
* Checks if a package name matches an exclusion pattern.
* Supports trailing wildcard (*) for prefix matching.
* @param {string} packageName
* @param {string} pattern
* @returns {boolean}
*/
export function matchesExclusionPattern(packageName, pattern) {
if (pattern.endsWith("/*")) {
return packageName.startsWith(pattern.slice(0, -1));
}
return packageName === pattern;
}
/**
* @param {string | undefined} packageName
* @returns {boolean}
*/
export function isExcludedFromMinimumPackageAge(packageName) {
if (!packageName) {
return false;
}
const exclusions = getMinimumPackageAgeExclusions();
const candidateNames = getEquivalentPackageNames(packageName, getEcoSystem());
return exclusions.some((pattern) =>
candidateNames.some((name) => matchesExclusionPattern(name, pattern))
);
}

62
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/npm/modifyNpmInfo.js
View file

@ -1,13 +1,7 @@
import {
getMinimumPackageAgeHours,
getNpmMinimumPackageAgeExclusions,
} from "../../../../config/settings.js";
import { getMinimumPackageAgeHours } from "../../../../config/settings.js";
import { ui } from "../../../../environment/userInteraction.js";
import { getHeaderValueAsString } from "../../http-utils.js";
const state = {
hasSuppressedVersions: false,
};
import { clearCachingHeaders, getHeaderValueAsString } from "../../http-utils.js";
import { recordSuppressedVersion } from "../suppressedVersionsState.js";
/**
* @param {NodeJS.Dict<string | string[]>} headers
@ -68,16 +62,6 @@ export function modifyNpmInfoResponse(body, headers) {
return body;
}
// Check if this package is excluded from minimum age filtering
const packageName = bodyJson.name;
const exclusions = getNpmMinimumPackageAgeExclusions();
if (packageName && exclusions.some((pattern) => matchesExclusionPattern(packageName, pattern))) {
ui.writeVerbose(
`Safe-chain: ${packageName} is excluded from minimum package age filtering (minimumPackageAgeExclusions setting).`
);
return body;
}
const cutOff = new Date(
new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
);
@ -95,15 +79,7 @@ export function modifyNpmInfoResponse(body, headers) {
const timestampValue = new Date(timestamp);
if (timestampValue > cutOff) {
deleteVersionFromJson(bodyJson, version);
if (headers) {
// When modifying the response, the etag and last-modified headers
// no longer match the content so they needs to be removed before sending the response.
delete headers["etag"];
delete headers["last-modified"];
// Removing the cache-control header will prevent the package manager from caching
// the modified response.
delete headers["cache-control"];
}
clearCachingHeaders(headers);
}
}
@ -127,7 +103,7 @@ export function modifyNpmInfoResponse(body, headers) {
* @param {string} version
*/
function deleteVersionFromJson(json, version) {
state.hasSuppressedVersions = true;
recordSuppressedVersion();
const packageName = typeof json?.name === "string" ? json.name : "(unknown)";
@ -185,22 +161,20 @@ function getMostRecentTag(tagList) {
}
/**
* @returns {boolean}
* @param {Buffer} body
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @returns {string | undefined}
*/
export function getHasSuppressedVersions() {
return state.hasSuppressedVersions;
}
export function getPackageNameFromMetadataResponse(body, headers) {
try {
const contentType = getHeaderValueAsString(headers, "content-type");
if (!contentType?.toLowerCase().includes("application/json")) {
return undefined;
}
/**
* Checks if a package name matches an exclusion pattern.
* Supports trailing wildcard (*) for prefix matching.
* @param {string} packageName
* @param {string} pattern
* @returns {boolean}
*/
function matchesExclusionPattern(packageName, pattern) {
if (pattern.endsWith("/*")) {
return packageName.startsWith(pattern.slice(0, -1));
const bodyJson = JSON.parse(body.toString("utf8"));
return typeof bodyJson.name === "string" ? bodyJson.name : undefined;
} catch {
return undefined;
}
return packageName === pattern;
}

49
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/npm/npmInterceptor.js
View file

@ -5,11 +5,16 @@ import {
import { isMalwarePackage } from "../../../../scanning/audit/index.js";
import { interceptRequests } from "../interceptorBuilder.js";
import {
getPackageNameFromMetadataResponse,
isPackageInfoUrl,
modifyNpmInfoRequestHeaders,
modifyNpmInfoResponse,
} from "./modifyNpmInfo.js";
import { parseNpmPackageUrl } from "./parseNpmPackageUrl.js";
import { openNewPackagesDatabase } from "../../../../scanning/newPackagesListCache.js";
import {
isExcludedFromMinimumPackageAge,
} from "../minimumPackageAgeExclusions.js";
const knownJsRegistries = [
"registry.npmjs.org",
@ -43,14 +48,54 @@ function buildNpmInterceptor(registry) {
reqContext.targetUrl,
registry
);
const minimumAgeChecksEnabled = !skipMinimumPackageAge();
if (await isMalwarePackage(packageName, version)) {
reqContext.blockMalware(packageName, version);
return;
}
if (!skipMinimumPackageAge() && isPackageInfoUrl(reqContext.targetUrl)) {
if (minimumAgeChecksEnabled && isPackageInfoUrl(reqContext.targetUrl)) {
reqContext.modifyRequestHeaders(modifyNpmInfoRequestHeaders);
reqContext.modifyBody(modifyNpmInfoResponse);
reqContext.modifyBody(modifyNpmInfoResponseUnlessExcluded);
return;
}
// For tarball requests the metadata check above is skipped, so we check the
// new packages list as a fallback (covers e.g. frozen-lockfile installs).
if (
minimumAgeChecksEnabled &&
packageName &&
version &&
!isExcludedFromMinimumPackageAge(packageName)
) {
const newPackagesDatabase = await openNewPackagesDatabase();
if (newPackagesDatabase.isNewlyReleasedPackage(packageName, version)) {
reqContext.blockMinimumAgeRequest(
packageName,
version,
`Forbidden - blocked by safe-chain direct download minimum package age (${packageName}@${version})`
);
}
}
});
}
/**
* @param {Buffer} body
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @returns {Buffer}
*/
function modifyNpmInfoResponseUnlessExcluded(body, headers) {
const metadataPackageName = getPackageNameFromMetadataResponse(body, headers);
if (
metadataPackageName &&
isExcludedFromMinimumPackageAge(metadataPackageName)
) {
return body;
}
return modifyNpmInfoResponse(body, headers);
}

76
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/npm/npmInterceptor.minPackageAge.spec.js
View file

@ -5,13 +5,25 @@ describe("npmInterceptor minimum package age", async () => {
let minimumPackageAgeSettings = 48;
let skipMinimumPackageAgeSetting = false;
let minimumPackageAgeExclusionsSetting = [];
let newlyReleasedPackages = new Set();
mock.module("../../../../config/settings.js", {
namedExports: {
ECOSYSTEM_JS: "js",
ECOSYSTEM_PY: "py",
getMinimumPackageAgeHours: () => minimumPackageAgeSettings,
skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
getNpmCustomRegistries: () => [],
getNpmMinimumPackageAgeExclusions: () => minimumPackageAgeExclusionsSetting,
getMinimumPackageAgeExclusions: () => minimumPackageAgeExclusionsSetting,
getEcoSystem: () => "js",
},
});
mock.module("../../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: (name, version) =>
newlyReleasedPackages.has(`${name}@${version}`),
}),
},
});
@ -359,6 +371,67 @@ describe("npmInterceptor minimum package age", async () => {
assert.equal(modifiedJson["dist-tags"]["latest"], "2.0.0");
});
it("Should suppress too-young versions on metadata requests without directly blocking the request", async () => {
minimumPackageAgeSettings = 5;
skipMinimumPackageAgeSetting = false;
const packageUrl = "https://registry.npmjs.org/lodash";
const interceptor = npmInterceptorForUrl(packageUrl);
const requestHandler = await interceptor.handleRequest(packageUrl);
assert.equal(requestHandler.blockResponse, undefined);
assert.equal(requestHandler.modifiesResponse(), true);
});
it("Should directly block tarball requests when the new packages list marks them as too young", async () => {
minimumPackageAgeSettings = 5;
skipMinimumPackageAgeSetting = false;
newlyReleasedPackages = new Set(["lodash@4.17.21"]);
const packageUrl =
"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz?integrity=sha512-abc123";
const interceptor = npmInterceptorForUrl(packageUrl);
const requestHandler = await interceptor.handleRequest(packageUrl);
assert.ok(requestHandler.blockResponse);
assert.equal(requestHandler.modifiesResponse(), false);
assert.equal(requestHandler.blockResponse.statusCode, 403);
assert.equal(
requestHandler.blockResponse.message,
"Forbidden - blocked by safe-chain direct download minimum package age (lodash@4.17.21)"
);
});
it("Should not block tarball requests when skipMinimumPackageAge is enabled", async () => {
minimumPackageAgeSettings = 5;
skipMinimumPackageAgeSetting = true;
minimumPackageAgeExclusionsSetting = [];
newlyReleasedPackages = new Set(["lodash@4.17.21"]);
const packageUrl =
"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz";
const interceptor = npmInterceptorForUrl(packageUrl);
const requestHandler = await interceptor.handleRequest(packageUrl);
assert.equal(requestHandler.blockResponse, undefined);
assert.equal(requestHandler.modifiesResponse(), false);
});
it("Should not block tarball requests when the package is excluded from minimum age", async () => {
minimumPackageAgeSettings = 5;
skipMinimumPackageAgeSetting = false;
minimumPackageAgeExclusionsSetting = ["lodash"];
newlyReleasedPackages = new Set(["lodash@4.17.21"]);
const packageUrl =
"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz";
const interceptor = npmInterceptorForUrl(packageUrl);
const requestHandler = await interceptor.handleRequest(packageUrl);
assert.equal(requestHandler.blockResponse, undefined);
assert.equal(requestHandler.modifiesResponse(), false);
});
it("Should not filter packages when package is in exclusion list", async () => {
minimumPackageAgeSettings = 5;
skipMinimumPackageAgeSetting = false;
@ -540,6 +613,7 @@ describe("npmInterceptor minimum package age", async () => {
minimumPackageAgeSettings = 5;
skipMinimumPackageAgeSetting = false;
minimumPackageAgeExclusionsSetting = []; // Reset to empty
newlyReleasedPackages = new Set();
const packageUrl = "https://registry.npmjs.org/lodash";

58
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/npm/npmInterceptor.packageDownload.spec.js
View file

@ -1,9 +1,11 @@
import { describe, it, mock } from "node:test";
import { describe, it, mock, beforeEach } from "node:test";
import assert from "node:assert";
let lastPackage;
let malwareResponse = false;
let customRegistries = [];
let newlyReleasedPackages = new Set();
let skipMinimumPackageAgeSetting = false;
mock.module("../../../../scanning/audit/index.js", {
namedExports: {
@ -26,14 +28,30 @@ mock.module("../../../../config/settings.js", {
setEcoSystem: () => {},
getMinimumPackageAgeHours: () => 24,
getNpmCustomRegistries: () => customRegistries,
getNpmMinimumPackageAgeExclusions: () => [],
skipMinimumPackageAge: () => false,
getMinimumPackageAgeExclusions: () => [],
skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
},
});
mock.module("../../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: (name, version) =>
newlyReleasedPackages.has(`${name}@${version}`),
}),
},
});
describe("npmInterceptor", async () => {
const { npmInterceptorForUrl } = await import("./npmInterceptor.js");
beforeEach(() => {
lastPackage = undefined;
malwareResponse = false;
customRegistries = [];
newlyReleasedPackages = new Set();
skipMinimumPackageAgeSetting = false;
});
const parserCases = [
// Regular packages
{
@ -109,6 +127,10 @@ describe("npmInterceptor", async () => {
url: "https://registry.yarnpkg.com/@babel/core/-/core-7.21.4.tgz",
expected: { packageName: "@babel/core", version: "7.21.4" },
},
{
url: "https://registry.yarnpkg.com/@music-i18n%2fverovio/-/verovio-1.4.1.tgz",
expected: { packageName: "@music-i18n/verovio", version: "1.4.1" },
},
// URL to get package info, not tarball
{
url: "https://registry.npmjs.org/lodash",
@ -178,6 +200,36 @@ describe("npmInterceptor", async () => {
"Block response should have correct status message"
);
});
it("should block direct tarball downloads for newly released packages", async () => {
const url =
"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz?integrity=sha512-abc123";
malwareResponse = false;
skipMinimumPackageAgeSetting = false;
newlyReleasedPackages = new Set(["lodash@4.17.21"]);
const interceptor = npmInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain direct download minimum package age (lodash@4.17.21)"
);
});
it("should not block direct tarball downloads when minimum age checks are skipped", async () => {
const url = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz";
malwareResponse = false;
skipMinimumPackageAgeSetting = true;
newlyReleasedPackages = new Set(["lodash@4.17.21"]);
const interceptor = npmInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
});
});
describe("npmInterceptor with custom registries", async () => {

23
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/npm/parseNpmPackageUrl.js
View file

@ -5,12 +5,29 @@
*/
export function parseNpmPackageUrl(url, registry) {
let packageName, version;
if (!registry || !url.endsWith(".tgz")) {
let parsedUrl;
try {
parsedUrl = new URL(url);
} catch {
return { packageName, version };
}
const registryIndex = url.indexOf(registry);
const afterRegistry = url.substring(registryIndex + registry.length + 1); // +1 to skip the slash
const pathname = parsedUrl.pathname;
if (!registry || !pathname.endsWith(".tgz")) {
return { packageName, version };
}
const registryPrefix = `${registry}/`;
const urlAfterProtocol = `${parsedUrl.host}${pathname}`;
if (!urlAfterProtocol.startsWith(registryPrefix)) {
return { packageName, version };
}
const afterRegistry = decodeURIComponent(
urlAfterProtocol.substring(registryPrefix.length)
);
const separatorIndex = afterRegistry.indexOf("/-/");
if (separatorIndex === -1) {

184
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/modifyPipInfo.js Normal file
View file

@ -0,0 +1,184 @@
import { ui } from "../../../../environment/userInteraction.js";
import { clearCachingHeaders } from "../../http-utils.js";
import { normalizePipPackageName } from "../../../../scanning/packageNameVariants.js";
import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
export { parsePipMetadataUrl, isPipPackageInfoUrl } from "./parsePipPackageUrl.js";
import { getPipMetadataContentType, logSuppressedVersion } from "./pipMetadataResponseUtils.js";
import { modifyPipJsonResponse } from "./modifyPipJsonResponse.js";
/**
* Strip conditional GET headers so PyPI always returns a full 200 response
* with a body we can rewrite. Without this, pip sends If-None-Match /
* If-Modified-Since, PyPI responds 304 Not Modified (empty body), and
* safe-chain cannot rewrite it — leaving pip with a cached index that still
* lists too-young versions. Those versions are then blocked at direct-download
* time with a hard 403, preventing dependency resolution from completing.
*
* @param {NodeJS.Dict<string | string[]>} headers
* @returns {NodeJS.Dict<string | string[]>}
*/
export function modifyPipInfoRequestHeaders(headers) {
delete headers["if-none-match"];
delete headers["if-modified-since"];
return headers;
}
// Match simple-index anchor tags and capture their href so we can suppress
// individual distribution links from PyPI HTML metadata responses.
const HTML_ANCHOR_HREF_RE =
/<a\b[^>]*href\s*=\s*(["'])([^"']+)\1[^>]*>[\s\S]*?<\/a>/gi;
/**
* @param {Buffer} body
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {Buffer}
*/
export function modifyPipInfoResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
) {
try {
const contentType = getPipMetadataContentType(headers);
if (!contentType || body.byteLength === 0) {
return body;
}
if (
contentType.includes("html") ||
contentType.includes("application/vnd.pypi.simple.v1+html")
) {
return modifyHtmlSimpleResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
}
if (
contentType.includes("json") ||
contentType.includes("application/vnd.pypi.simple.v1+json")
) {
return modifyJsonResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
}
return body;
} catch (/** @type {any} */ err) {
ui.writeVerbose(
`Safe-chain: PyPI package metadata not in expected format - bypassing modification. Error: ${err.message}`
);
return body;
}
}
/**
* @param {Buffer} body
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {Buffer}
*/
function modifyHtmlSimpleResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
) {
const html = body.toString("utf8");
let modified = false;
const rewriteHtmlAnchor = createHtmlAnchorRewriter(
metadataUrl,
isNewlyReleasedPackage,
packageName,
() => {
modified = true;
}
);
const updatedHtml = html.replace(HTML_ANCHOR_HREF_RE, rewriteHtmlAnchor);
if (!modified) return body;
const modifiedBuffer = Buffer.from(updatedHtml);
clearCachingHeaders(headers);
return modifiedBuffer;
}
/**
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @param {() => void} onModified
* @returns {(anchor: string, quote: string, href: string) => string}
*/
function createHtmlAnchorRewriter(
metadataUrl,
isNewlyReleasedPackage,
packageName,
onModified
) {
return (anchor, _quote, href) => {
const resolvedHref = new URL(href, metadataUrl).toString();
const { packageName: hrefPackageName, version } = parsePipPackageFromUrl(
resolvedHref,
new URL(resolvedHref).host
);
if (
hrefPackageName &&
normalizePipPackageName(hrefPackageName) ===
normalizePipPackageName(packageName) &&
version &&
isNewlyReleasedPackage(packageName, version)
) {
onModified();
logSuppressedVersion(packageName, version);
return "";
}
return anchor;
};
}
/**
* @param {Buffer} body
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {Buffer}
*/
function modifyJsonResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
) {
const json = JSON.parse(body.toString("utf8"));
const modified = modifyPipJsonResponse(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
if (!modified) return body;
const modifiedBuffer = Buffer.from(JSON.stringify(json));
clearCachingHeaders(headers);
return modifiedBuffer;
}

302
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/modifyPipInfo.spec.js Normal file
View file

@ -0,0 +1,302 @@
import { describe, it, mock } from "node:test";
import assert from "node:assert";
describe("modifyPipInfo", async () => {
mock.module("../../../../config/settings.js", {
namedExports: {
getMinimumPackageAgeHours: () => 48,
ECOSYSTEM_PY: "py",
},
});
mock.module("../../../../environment/userInteraction.js", {
namedExports: {
ui: {
writeVerbose: () => {},
},
},
});
const {
modifyPipInfoResponse,
} = await import("./modifyPipInfo.js");
it("removes too-young files from simple HTML metadata", () => {
const headers = {
"content-type": "application/vnd.pypi.simple.v1+html",
etag: "abc",
"cache-control": "public",
"content-length": "999",
"transfer-encoding": "chunked",
};
const body = Buffer.from(`
<!doctype html>
<html>
<body>
<a href="https://files.pythonhosted.org/packages/source/r/requests/requests-1.0.0.tar.gz">requests-1.0.0.tar.gz</a>
<a href="https://files.pythonhosted.org/packages/source/r/requests/requests-2.0.0.tar.gz">requests-2.0.0.tar.gz</a>
</body>
</html>
`);
const modified = modifyPipInfoResponse(
body,
headers,
"https://pypi.org/simple/requests/",
(_packageName, version) => version === "2.0.0",
"requests"
).toString("utf8");
assert.ok(modified.includes("requests-1.0.0.tar.gz"));
assert.ok(!modified.includes("requests-2.0.0.tar.gz"));
assert.equal(headers.etag, undefined);
assert.equal(headers["cache-control"], undefined);
assert.equal(headers["content-length"], undefined);
assert.equal(headers["transfer-encoding"], "chunked");
});
it("leaves mixed-case transport headers untouched for MITM layer to normalize", () => {
const headers = {
"content-type": "application/json",
ETag: "abc",
"Content-Length": "999",
"Last-Modified": "yesterday",
"Cache-Control": "public, max-age=60",
"Transfer-Encoding": "chunked",
};
const body = Buffer.from(
JSON.stringify({
info: { version: "2.0.0" },
releases: {
"1.0.0": [{ filename: "requests-1.0.0.tar.gz" }],
"2.0.0": [{ filename: "requests-2.0.0.tar.gz" }],
},
})
);
modifyPipInfoResponse(
body,
headers,
"https://pypi.org/pypi/requests/json",
(_packageName, version) => version === "2.0.0",
"requests"
);
assert.equal(headers.ETag, "abc");
assert.equal(headers["Last-Modified"], "yesterday");
assert.equal(headers["Cache-Control"], "public, max-age=60");
assert.equal(headers["Transfer-Encoding"], "chunked");
assert.equal(headers["Content-Length"], "999");
assert.equal(headers["content-length"], undefined);
});
it("returns body unchanged when no HTML versions are suppressed", () => {
const headers = {
"content-type": "application/vnd.pypi.simple.v1+html",
etag: "abc",
};
const body = Buffer.from(
`<a href="https://files.pythonhosted.org/packages/source/r/requests/requests-1.0.0.tar.gz">requests-1.0.0.tar.gz</a>`
);
const result = modifyPipInfoResponse(
body,
headers,
"https://pypi.org/simple/requests/",
() => false,
"requests"
);
assert.equal(result, body); // same Buffer reference — no copy made
assert.equal(headers.etag, "abc"); // headers untouched
});
it("matches HTML anchor hrefs using normalised package name (underscore vs hyphen)", () => {
const headers = { "content-type": "application/vnd.pypi.simple.v1+html" };
const body = Buffer.from(
`<a href="https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0.tar.gz">foo_bar-2.0.0.tar.gz</a>` +
`<a href="https://files.pythonhosted.org/packages/xx/yy/foo_bar-1.0.0.tar.gz">foo_bar-1.0.0.tar.gz</a>`
);
const modified = modifyPipInfoResponse(
body,
headers,
"https://pypi.org/simple/foo-bar/",
(_packageName, version) => version === "2.0.0",
"foo-bar" // hyphenated name, hrefs use underscore
).toString("utf8");
assert.ok(!modified.includes("foo_bar-2.0.0.tar.gz"));
assert.ok(modified.includes("foo_bar-1.0.0.tar.gz"));
});
it("matches anchor href regex with single quotes and extra attributes", () => {
const headers = { "content-type": "application/vnd.pypi.simple.v1+html" };
const body = Buffer.from(`
<a
data-requires-python="&gt;=3.9"
class="pkg"
href='https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0.tar.gz'
>
foo_bar-2.0.0.tar.gz
</a>
<a href="https://files.pythonhosted.org/packages/xx/yy/foo_bar-1.0.0.tar.gz">foo_bar-1.0.0.tar.gz</a>
`);
const modified = modifyPipInfoResponse(
body,
headers,
"https://pypi.org/simple/foo-bar/",
(_packageName, version) => version === "2.0.0",
"foo-bar"
).toString("utf8");
assert.ok(!modified.includes("foo_bar-2.0.0.tar.gz"));
assert.ok(modified.includes("foo_bar-1.0.0.tar.gz"));
});
it("removes too-young files from simple JSON metadata", () => {
const headers = {
"content-type": "application/vnd.pypi.simple.v1+json",
};
const body = Buffer.from(
JSON.stringify({
name: "requests",
files: [
{
filename: "requests-1.0.0.tar.gz",
url: "https://files.pythonhosted.org/packages/source/r/requests/requests-1.0.0.tar.gz",
},
{
filename: "requests-2.0.0.tar.gz",
url: "https://files.pythonhosted.org/packages/source/r/requests/requests-2.0.0.tar.gz",
},
],
})
);
const modified = JSON.parse(
modifyPipInfoResponse(
body,
headers,
"https://pypi.org/simple/requests/",
(_packageName, version) => version === "2.0.0",
"requests"
).toString("utf8")
);
assert.equal(modified.files.length, 1);
assert.equal(modified.files[0].filename, "requests-1.0.0.tar.gz");
});
it("filters simple JSON metadata entries that have only filename (no url)", () => {
const headers = { "content-type": "application/vnd.pypi.simple.v1+json" };
const body = Buffer.from(
JSON.stringify({
name: "requests",
files: [
{ filename: "requests-1.0.0.tar.gz" },
{ filename: "requests-2.0.0.tar.gz" },
],
})
);
const modified = JSON.parse(
modifyPipInfoResponse(
body,
headers,
"https://pypi.org/simple/requests/",
(_packageName, version) => version === "2.0.0",
"requests"
).toString("utf8")
);
assert.equal(modified.files.length, 1);
assert.equal(modified.files[0].filename, "requests-1.0.0.tar.gz");
});
it("recalculates JSON API info.version after removing too-young releases", () => {
const headers = {
"content-type": "application/json",
};
const body = Buffer.from(
JSON.stringify({
info: { version: "2.0.0" },
releases: {
"1.0.0": [
{
filename: "requests-1.0.0.tar.gz",
upload_time_iso_8601: "2024-01-01T00:00:00.000Z",
},
],
"2.0.0": [
{
filename: "requests-2.0.0.tar.gz",
upload_time_iso_8601: "2024-01-02T00:00:00.000Z",
},
],
"3.0.0rc1": [
{
filename: "requests-3.0.0rc1.tar.gz",
upload_time_iso_8601: "2024-01-03T00:00:00.000Z",
},
],
},
urls: [
{ filename: "requests-2.0.0.tar.gz" },
],
})
);
const modified = JSON.parse(
modifyPipInfoResponse(
body,
headers,
"https://pypi.org/pypi/requests/json",
(_packageName, version) =>
version === "2.0.0" || version === "3.0.0rc1",
"requests"
).toString("utf8")
);
assert.deepEqual(Object.keys(modified.releases), ["1.0.0"]);
assert.equal(modified.info.version, "1.0.0");
assert.equal(modified.urls.length, 0);
});
it("falls back to latest pre-release when all stable versions are removed", () => {
const headers = { "content-type": "application/json" };
const body = Buffer.from(
JSON.stringify({
info: { version: "2.0.0rc2" },
releases: {
"1.0.0rc1": [{ filename: "requests-1.0.0rc1.tar.gz" }],
"2.0.0rc2": [{ filename: "requests-2.0.0rc2.tar.gz" }],
},
urls: [],
})
);
const modified = JSON.parse(
modifyPipInfoResponse(
body,
headers,
"https://pypi.org/pypi/requests/json",
(_packageName, version) => version === "2.0.0rc2",
"requests"
).toString("utf8")
);
assert.deepEqual(Object.keys(modified.releases), ["1.0.0rc1"]);
assert.equal(modified.info.version, "1.0.0rc1");
});
});

176
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/modifyPipJsonResponse.js Normal file
View file

@ -0,0 +1,176 @@
import {
calculateLatestVersion,
getAvailableVersionsFromJson,
getPackageVersionFromMetadataFile,
} from "./pipMetadataVersionUtils.js";
import { logSuppressedVersion } from "./pipMetadataResponseUtils.js";
/**
* @param {any} json
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
*/
export function modifyPipJsonResponse(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
) {
const filesModified = filterJsonMetadataFiles(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
const releasesModified = removeJsonMetadataReleases(
json,
isNewlyReleasedPackage,
packageName
);
const urlsModified = filterJsonMetadataUrls(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
const versionModified = updateJsonInfoVersion(json, metadataUrl);
return filesModified || releasesModified || urlsModified || versionModified;
}
/**
* @param {any} json
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
*/
function filterJsonMetadataFiles(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
) {
if (!Array.isArray(json.files)) {
return false;
}
let modified = false;
const loggedVersions = new Set();
json.files = json.files.filter((/** @type {any} */ file) => {
const version = getPackageVersionFromMetadataFile(file, metadataUrl);
if (version && isNewlyReleasedPackage(packageName, version)) {
modified = true;
if (!loggedVersions.has(version)) {
logSuppressedVersion(packageName, version);
loggedVersions.add(version);
}
return false;
}
return true;
});
return modified;
}
/**
* @param {any} json
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
*/
function removeJsonMetadataReleases(json, isNewlyReleasedPackage, packageName) {
if (!json.releases || typeof json.releases !== "object") {
return false;
}
let modified = false;
for (const [version, files] of Object.entries(json.releases)) {
if (
Array.isArray(/** @type {unknown[]} */ (files)) &&
isNewlyReleasedPackage(packageName, version)
) {
delete json.releases[version];
modified = true;
logSuppressedVersion(packageName, version);
}
}
return modified;
}
/**
* @param {any} json
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
*/
function filterJsonMetadataUrls(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
) {
if (!Array.isArray(json.urls)) {
return false;
}
let modified = false;
const loggedVersions = new Set();
json.urls = json.urls.filter((/** @type {any} */ file) => {
const version = getPackageVersionFromMetadataFile(file, metadataUrl);
if (version && isNewlyReleasedPackage(packageName, version)) {
modified = true;
if (!loggedVersions.has(version)) {
logSuppressedVersion(packageName, version);
loggedVersions.add(version);
}
return false;
}
return true;
});
return modified;
}
/**
* @param {any} json
* @param {string} metadataUrl
* @returns {boolean}
*/
function updateJsonInfoVersion(json, metadataUrl) {
if (!json.info || typeof json.info !== "object") {
return false;
}
const replacementVersion = computeReplacementVersion(json, metadataUrl);
if (
typeof json.info.version !== "string" ||
!replacementVersion ||
json.info.version === replacementVersion
) {
return false;
}
json.info.version = replacementVersion;
return true;
}
/**
* @param {any} json
* @param {string} metadataUrl
* @returns {string | undefined}
*/
function computeReplacementVersion(json, metadataUrl) {
const candidateVersions = getAvailableVersionsFromJson(json, metadataUrl);
return calculateLatestVersion(candidateVersions);
}

162
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/parsePipPackageUrl.js Normal file
View file

@ -0,0 +1,162 @@
/**
* Parses a PyPI metadata URL and returns the package name and API type.
*
* @example
* parsePipMetadataUrl("https://pypi.org/simple/requests/")
* // => { packageName: "requests", type: "simple" }
*
* parsePipMetadataUrl("https://pypi.org/pypi/requests/json")
* // => { packageName: "requests", type: "json" }
*
* parsePipMetadataUrl("https://pypi.org/pypi/requests/2.28.1/json")
* // => { packageName: "requests", type: "json" }
*
* parsePipMetadataUrl("https://files.pythonhosted.org/packages/requests-2.28.1.tar.gz")
* // => { packageName: undefined, type: undefined }
*
* @param {string} url
* @returns {{ packageName: string | undefined, type: "simple" | "json" | undefined }}
*/
export function parsePipMetadataUrl(url) {
if (typeof url !== "string") {
return { packageName: undefined, type: undefined };
}
let urlObj;
try {
urlObj = new URL(url);
} catch {
return { packageName: undefined, type: undefined };
}
const pathSegments = urlObj.pathname.split("/").filter(Boolean);
if (pathSegments[0] === "simple" && pathSegments[1]) {
return {
packageName: decodeURIComponent(pathSegments[1]),
type: "simple",
};
}
if (
pathSegments[0] === "pypi" &&
pathSegments[pathSegments.length - 1] === "json" &&
pathSegments[1]
) {
return {
packageName: decodeURIComponent(pathSegments[1]),
type: "json",
};
}
return { packageName: undefined, type: undefined };
}
/**
* @param {string} url
* @returns {boolean}
*/
export function isPipPackageInfoUrl(url) {
return !!parsePipMetadataUrl(url).packageName;
}
/**
* Parse Python package artifact URLs from PyPI-style registries.
* Examples:
* - Wheel: https://files.pythonhosted.org/packages/.../requests-2.28.1-py3-none-any.whl
* - Wheel metadata: https://files.pythonhosted.org/packages/.../requests-2.28.1-py3-none-any.whl.metadata
* - Sdist: https://files.pythonhosted.org/packages/.../requests-2.28.1.tar.gz
*
* @param {string} url
* @param {string} registry
* @returns {{packageName: string | undefined, version: string | undefined}}
*/
export function parsePipPackageFromUrl(url, registry) {
if (!registry || typeof url !== "string") {
return { packageName: undefined, version: undefined };
}
let urlObj;
try {
urlObj = new URL(url);
} catch {
return { packageName: undefined, version: undefined };
}
const lastSegment = urlObj.pathname.split("/").filter(Boolean).pop();
if (!lastSegment) {
return { packageName: undefined, version: undefined };
}
const filename = decodeURIComponent(lastSegment);
const wheelExtRe = /\.whl(?:\.metadata)?$/;
if (wheelExtRe.test(filename)) {
return parseWheelFilename(filename, wheelExtRe);
}
const sdistExtWithMetadataRe = /\.(tar\.gz|zip|tar\.bz2|tar\.xz)(\.metadata)?$/i;
if (!sdistExtWithMetadataRe.test(filename)) {
return { packageName: undefined, version: undefined };
}
return parseSdistFilename(filename, sdistExtWithMetadataRe);
}
/**
* Parse wheel filenames and Poetry preflight metadata.
* Examples:
* - foo_bar-2.0.0-py3-none-any.whl
* - foo_bar-2.0.0-py3-none-any.whl.metadata
*
* @param {string} filename
* @param {RegExp} wheelExtRe
* @returns {{packageName: string | undefined, version: string | undefined}}
*/
function parseWheelFilename(filename, wheelExtRe) {
const base = filename.replace(wheelExtRe, "");
const firstDash = base.indexOf("-");
if (firstDash <= 0) {
return { packageName: undefined, version: undefined };
}
const packageName = base.slice(0, firstDash);
const rest = base.slice(firstDash + 1);
const secondDash = rest.indexOf("-");
const version = secondDash >= 0 ? rest.slice(0, secondDash) : rest;
// "latest" is a resolver-style token, not an actual published artifact version.
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
/**
* Parse source distribution filenames, with optional metadata suffix.
* Examples:
* - requests-2.28.1.tar.gz
* - requests-2.28.1.zip
* - requests-2.28.1.tar.gz.metadata
*
* @param {string} filename
* @param {RegExp} sdistExtWithMetadataRe
* @returns {{packageName: string | undefined, version: string | undefined}}
*/
function parseSdistFilename(filename, sdistExtWithMetadataRe) {
const base = filename.replace(sdistExtWithMetadataRe, "");
const lastDash = base.lastIndexOf("-");
if (lastDash <= 0 || lastDash >= base.length - 1) {
return { packageName: undefined, version: undefined };
}
const packageName = base.slice(0, lastDash);
const version = base.slice(lastDash + 1);
// "latest" is a resolver-style token, not an actual published artifact version.
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}

100
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/parsePipPackageUrl.spec.js Normal file
View file

@ -0,0 +1,100 @@
import { describe, it } from "node:test";
import assert from "node:assert";
import {
isPipPackageInfoUrl,
parsePipMetadataUrl,
parsePipPackageFromUrl,
} from "./parsePipPackageUrl.js";
describe("parsePipPackageUrl", () => {
it("parses simple metadata URLs", () => {
assert.deepEqual(parsePipMetadataUrl("https://pypi.org/simple/requests/"), {
packageName: "requests",
type: "simple",
});
});
it("parses json metadata URLs", () => {
assert.deepEqual(parsePipMetadataUrl("https://pypi.org/pypi/requests/json"), {
packageName: "requests",
type: "json",
});
});
it("parses per-version json metadata URLs", () => {
assert.deepEqual(
parsePipMetadataUrl("https://pypi.org/pypi/requests/2.28.1/json"),
{ packageName: "requests", type: "json" }
);
});
it("decodes encoded metadata package names", () => {
assert.deepEqual(
parsePipMetadataUrl("https://pypi.org/simple/foo-bar%5Fbaz/"),
{
packageName: "foo-bar_baz",
type: "simple",
}
);
});
it("returns undefined for unrecognized metadata paths", () => {
assert.deepEqual(
parsePipMetadataUrl("https://pypi.org/unknown/requests/"),
{
packageName: undefined,
type: undefined,
}
);
});
it("returns undefined for invalid metadata URLs", () => {
assert.deepEqual(parsePipMetadataUrl("not a url"), {
packageName: undefined,
type: undefined,
});
});
it("recognizes package info URLs", () => {
assert.equal(
isPipPackageInfoUrl("https://pypi.org/simple/requests/"),
true
);
});
it("does not treat artifact URLs as package info URLs", () => {
assert.equal(
isPipPackageInfoUrl(
"https://files.pythonhosted.org/packages/source/r/requests/requests-2.28.1.tar.gz"
),
false
);
});
it("parses wheel artifact URLs", () => {
assert.deepEqual(
parsePipPackageFromUrl(
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl",
"files.pythonhosted.org"
),
{ packageName: "foo_bar", version: "2.0.0" }
);
});
it("parses sdist artifact URLs", () => {
assert.deepEqual(
parsePipPackageFromUrl(
"https://files.pythonhosted.org/packages/source/r/requests/requests-2.28.1.tar.gz",
"files.pythonhosted.org"
),
{ packageName: "requests", version: "2.28.1" }
);
});
it("returns undefined for non-artifact URLs", () => {
assert.deepEqual(
parsePipPackageFromUrl("https://pypi.org/simple/requests/", "pypi.org"),
{ packageName: undefined, version: undefined }
);
});
});

135
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pipInterceptor.pipCustomRegistries.spec.js → packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipInterceptor.customRegistries.spec.js
View file

@ -2,20 +2,36 @@ import { describe, it, mock } from "node:test";
import assert from "node:assert";
describe("pipInterceptor custom registries", async () => {
let lastPackage;
let scannedPackages;
let malwareResponse = false;
let customRegistries = [];
mock.module("../../../config/settings.js", {
mock.module("../../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getLoggingLevel: () => "silent",
getMinimumPackageAgeHours: () => 48,
getMinimumPackageAgeExclusions: () => [],
getPipCustomRegistries: () => customRegistries,
LOGGING_SILENT: "silent",
LOGGING_VERBOSE: "verbose",
skipMinimumPackageAge: () => false,
},
});
mock.module("../../../scanning/audit/index.js", {
mock.module("../../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: () => false,
}),
},
});
mock.module("../../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async (packageName, version) => {
lastPackage = { packageName, version };
scannedPackages.push({ packageName, version });
return malwareResponse;
},
},
@ -30,42 +46,45 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.ok(
interceptor,
"Interceptor should be created for custom registry"
);
assert.ok(interceptor);
});
it("should parse package from custom registry URL", async () => {
scannedPackages = [];
customRegistries = ["my-custom-registry.example.com"];
const url =
"https://my-custom-registry.example.com/packages/xx/yy/foobar-1.2.3.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
assert.deepEqual(lastPackage, {
packageName: "foobar",
version: "1.2.3",
});
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === "foobar" && version === "1.2.3"
)
);
});
it("should parse wheel package from custom registry URL", async () => {
scannedPackages = [];
customRegistries = ["private-pypi.internal.com"];
const url =
"https://private-pypi.internal.com/packages/foo_bar-2.0.0-py3-none-any.whl";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
assert.deepEqual(lastPackage, {
packageName: "foo-bar",
version: "2.0.0",
});
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === "foo-bar" && version === "2.0.0"
)
);
});
it("should handle multiple custom registries", async () => {
@ -82,14 +101,12 @@ describe("pipInterceptor custom registries", async () => {
const interceptor1 = pipInterceptorForUrl(url1);
const interceptor2 = pipInterceptorForUrl(url2);
assert.ok(interceptor1, "Interceptor should be created for first registry");
assert.ok(
interceptor2,
"Interceptor should be created for second registry"
);
assert.ok(interceptor1);
assert.ok(interceptor2);
});
it("should block malicious package from custom registry", async () => {
scannedPackages = [];
customRegistries = ["my-custom-registry.example.com"];
malwareResponse = true;
@ -97,26 +114,19 @@ describe("pipInterceptor custom registries", async () => {
"https://my-custom-registry.example.com/packages/malicious_package-1.0.0.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse, "Should contain a blockResponse");
assert.equal(
result.blockResponse.statusCode,
403,
"Block response should have status code 403"
);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain",
"Block response should have correct status message"
);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(result.blockResponse.message, "Forbidden - blocked by safe-chain");
malwareResponse = false;
});
it("should still work with known registries when custom registries are set", async () => {
scannedPackages = [];
customRegistries = ["my-custom-registry.example.com"];
const url =
@ -124,17 +134,16 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.ok(
interceptor,
"Interceptor should be created for known registry even with custom registries set"
);
assert.ok(interceptor);
await interceptor.handleRequest(url);
assert.deepEqual(lastPackage, {
packageName: "foobar",
version: "1.2.3",
});
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === "foobar" && version === "1.2.3"
)
);
});
it("should not create interceptor for unknown registry when custom registries are set", () => {
@ -143,11 +152,7 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.equal(
interceptor,
undefined,
"Interceptor should be undefined for unknown registry"
);
assert.equal(interceptor, undefined);
});
it("should handle empty custom registries array", () => {
@ -157,42 +162,44 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.equal(
interceptor,
undefined,
"Interceptor should be undefined when no custom registries are configured"
);
assert.equal(interceptor, undefined);
});
it("should parse .whl.metadata from custom registry", async () => {
scannedPackages = [];
customRegistries = ["private-pypi.internal.com"];
const url =
"https://private-pypi.internal.com/packages/foo_bar-2.0.0-py3-none-any.whl.metadata";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
assert.deepEqual(lastPackage, {
packageName: "foo-bar",
version: "2.0.0",
});
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === "foo-bar" && version === "2.0.0"
)
);
});
it("should parse .tar.gz.metadata from custom registry", async () => {
scannedPackages = [];
customRegistries = ["private-pypi.internal.com"];
const url =
"https://private-pypi.internal.com/packages/foo_bar-2.0.0.tar.gz.metadata";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
assert.deepEqual(lastPackage, {
packageName: "foo-bar",
version: "2.0.0",
});
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === "foo-bar" && version === "2.0.0"
)
);
});
});
});

124
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipInterceptor.js Normal file
View file

@ -0,0 +1,124 @@
import {
ECOSYSTEM_PY,
getPipCustomRegistries,
skipMinimumPackageAge,
} from "../../../../config/settings.js";
import { isMalwarePackage } from "../../../../scanning/audit/index.js";
import { getEquivalentPackageNames } from "../../../../scanning/packageNameVariants.js";
import { openNewPackagesDatabase } from "../../../../scanning/newPackagesListCache.js";
import { interceptRequests } from "../interceptorBuilder.js";
import { isExcludedFromMinimumPackageAge } from "../minimumPackageAgeExclusions.js";
import {
modifyPipInfoRequestHeaders,
modifyPipInfoResponse,
parsePipMetadataUrl,
} from "./modifyPipInfo.js";
import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
const knownPipRegistries = [
"files.pythonhosted.org",
"pypi.org",
"pypi.python.org",
"pythonhosted.org",
];
/**
* @param {string} url
* @returns {import("../interceptorBuilder.js").Interceptor | undefined}
*/
export function pipInterceptorForUrl(url) {
const customRegistries = getPipCustomRegistries();
const registries = [...knownPipRegistries, ...customRegistries];
const registry = registries.find((reg) => url.includes(reg));
if (registry) {
return buildPipInterceptor(registry);
}
return undefined;
}
/**
* @param {string} registry
* @returns {import("../interceptorBuilder.js").Interceptor | undefined}
*/
function buildPipInterceptor(registry) {
return interceptRequests(createPipRequestHandler(registry));
}
/**
* @param {string} registry
* @returns {(reqContext: import("../interceptorBuilder.js").RequestInterceptionContext) => Promise<void>}
*/
function createPipRequestHandler(registry) {
return async (reqContext) => {
const minimumAgeChecksEnabled = !skipMinimumPackageAge();
const metadataInfo = parsePipMetadataUrl(reqContext.targetUrl);
const metadataPackageName = metadataInfo.packageName;
if (
minimumAgeChecksEnabled &&
metadataPackageName &&
!isExcludedFromMinimumPackageAge(metadataPackageName)
) {
const newPackagesDatabase = await openNewPackagesDatabase();
reqContext.modifyRequestHeaders(modifyPipInfoRequestHeaders);
reqContext.modifyBody((body, headers) =>
modifyPipInfoResponse(
body,
headers,
reqContext.targetUrl,
newPackagesDatabase.isNewlyReleasedPackage,
metadataPackageName
)
);
return;
}
const { packageName, version } = parsePipPackageFromUrl(
reqContext.targetUrl,
registry
);
if (!packageName) {
return;
}
const equivalentPackageNames = getEquivalentPackageNames(
packageName,
ECOSYSTEM_PY
);
let isMalicious = false;
for (const equivalentPackageName of equivalentPackageNames) {
if (await isMalwarePackage(equivalentPackageName, version)) {
isMalicious = true;
break;
}
}
if (isMalicious) {
reqContext.blockMalware(packageName, version);
return;
}
if (
version &&
minimumAgeChecksEnabled &&
!isExcludedFromMinimumPackageAge(packageName)
) {
const newPackagesDatabase = await openNewPackagesDatabase();
const isNewlyReleased = newPackagesDatabase.isNewlyReleasedPackage(
packageName,
version
);
if (isNewlyReleased) {
reqContext.blockMinimumAgeRequest(
packageName,
version,
`Forbidden - blocked by safe-chain direct download minimum package age (${packageName}@${version})`
);
}
}
};
}

168
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipInterceptor.minPackageAge.spec.js Normal file
View file

@ -0,0 +1,168 @@
import { describe, it, mock } from "node:test";
import assert from "node:assert";
describe("pipInterceptor minimum package age", async () => {
let skipMinimumPackageAgeSetting = false;
let newlyReleasedPackageResponse = false;
let minimumPackageAgeExclusionsSetting = [];
mock.module("../../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async () => false,
},
});
mock.module("../../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: (packageName, version) => {
return newlyReleasedPackageResponse &&
(packageName === "foo-bar" ||
packageName === "foo_bar" ||
packageName === "foo.bar") &&
version === "2.0.0";
},
}),
},
});
mock.module("../../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getLoggingLevel: () => "silent",
getMinimumPackageAgeHours: () => 48,
getMinimumPackageAgeExclusions: () => minimumPackageAgeExclusionsSetting,
getPipCustomRegistries: () => [],
LOGGING_SILENT: "silent",
LOGGING_VERBOSE: "verbose",
skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
},
});
const { pipInterceptorForUrl } = await import("./pipInterceptor.js");
it("should block newly released package downloads", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl";
newlyReleasedPackageResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain direct download minimum package age (foo_bar@2.0.0)"
);
newlyReleasedPackageResponse = false;
});
it("should modify simple metadata responses to suppress too-young versions", async () => {
const url = "https://pypi.org/simple/foo-bar/";
newlyReleasedPackageResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.modifiesResponse(), true);
const modifiedBody = result.modifyBody(
Buffer.from(`
<a href="https://files.pythonhosted.org/packages/xx/yy/foo_bar-1.0.0.tar.gz">foo_bar-1.0.0.tar.gz</a>
<a href="https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0.tar.gz">foo_bar-2.0.0.tar.gz</a>
`),
{
"content-type": "application/vnd.pypi.simple.v1+html",
}
).toString("utf8");
assert.ok(modifiedBody.includes("foo_bar-1.0.0.tar.gz"));
assert.ok(!modifiedBody.includes("foo_bar-2.0.0.tar.gz"));
newlyReleasedPackageResponse = false;
});
it("should not block newly released package downloads when skipMinimumPackageAge is enabled", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl";
newlyReleasedPackageResponse = true;
skipMinimumPackageAgeSetting = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
skipMinimumPackageAgeSetting = false;
newlyReleasedPackageResponse = false;
});
it("should not block newly released package downloads when the package is excluded", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl";
newlyReleasedPackageResponse = true;
minimumPackageAgeExclusionsSetting = ["foo-bar"];
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
minimumPackageAgeExclusionsSetting = [];
newlyReleasedPackageResponse = false;
});
it("should not modify metadata responses when the package is excluded", async () => {
const url = "https://pypi.org/simple/foo-bar/";
newlyReleasedPackageResponse = true;
minimumPackageAgeExclusionsSetting = ["foo-bar"];
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.modifiesResponse(), false);
minimumPackageAgeExclusionsSetting = [];
newlyReleasedPackageResponse = false;
});
it("strips If-None-Match and If-Modified-Since from metadata requests to prevent 304 cache bypass", async () => {
const url = "https://pypi.org/simple/foo-bar/";
newlyReleasedPackageResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
const headers = {
"if-none-match": '"some-etag"',
"if-modified-since": "Thu, 01 Jan 2026 00:00:00 GMT",
accept: "*/*",
};
result.modifyRequestHeaders(headers);
assert.equal(headers["if-none-match"], undefined, "If-None-Match must be stripped");
assert.equal(headers["if-modified-since"], undefined, "If-Modified-Since must be stripped");
assert.equal(headers.accept, "*/*", "unrelated headers must be preserved");
newlyReleasedPackageResponse = false;
});
it("should not block newly released package downloads when a dot-name package matches a hyphen exclusion", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo.bar-2.0.0.tar.gz";
newlyReleasedPackageResponse = true;
minimumPackageAgeExclusionsSetting = ["foo-bar"];
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
minimumPackageAgeExclusionsSetting = [];
newlyReleasedPackageResponse = false;
});
});

163
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipInterceptor.packageDownload.spec.js Normal file
View file

@ -0,0 +1,163 @@
import { describe, it, mock } from "node:test";
import assert from "node:assert";
describe("pipInterceptor", async () => {
let scannedPackages;
let malwareResponse = false;
mock.module("../../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async (packageName, version) => {
scannedPackages.push({ packageName, version });
return malwareResponse;
},
},
});
mock.module("../../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: () => false,
}),
},
});
mock.module("../../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getLoggingLevel: () => "silent",
getMinimumPackageAgeHours: () => 48,
getMinimumPackageAgeExclusions: () => [],
getPipCustomRegistries: () => [],
LOGGING_SILENT: "silent",
LOGGING_VERBOSE: "verbose",
skipMinimumPackageAge: () => false,
},
});
const { pipInterceptorForUrl } = await import("./pipInterceptor.js");
const parserCases = [
{
url: "https://files.pythonhosted.org/packages/xx/yy/foobar-1.2.3.tar.gz",
expected: { packageName: "foobar", version: "1.2.3" },
},
{
url: "https://pypi.org/packages/source/f/foobar/foobar-1.2.3.tar.gz",
expected: { packageName: "foobar", version: "1.2.3" },
},
{
url: "https://pypi.org/packages/source/f/foo-bar/foo-bar-0.9.0.tar.gz",
expected: { packageName: "foo-bar", version: "0.9.0" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0-py3-none-any.whl",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl.metadata",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
url: "https://pypi.org/packages/source/f/foo.bar/foo.bar-1.0.0.tar.gz",
expected: { packageName: "foo.bar", version: "1.0.0" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0b1.tar.gz",
expected: { packageName: "foo-bar", version: "2.0.0b1" },
},
{
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0.tar.gz.metadata",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0rc1.tar.gz",
expected: { packageName: "foo-bar", version: "2.0.0rc1" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0.post1.tar.gz",
expected: { packageName: "foo-bar", version: "2.0.0.post1" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0.dev1.tar.gz",
expected: { packageName: "foo-bar", version: "2.0.0.dev1" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0a1.tar.gz",
expected: { packageName: "foo-bar", version: "2.0.0a1" },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0-cp38-cp38-manylinux1_x86_64.whl",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
url: "https://pypi.org/simple/",
expected: { packageName: undefined, version: undefined },
},
{
url: "https://pypi.org/project/foobar/",
expected: { packageName: undefined, version: undefined },
},
{
url: "https://files.pythonhosted.org/packages/xx/yy/foobar-latest.tar.gz",
expected: { packageName: undefined, version: undefined },
},
{
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-latest.tar.gz",
expected: { packageName: undefined, version: undefined },
},
];
parserCases.forEach(({ url, expected }, index) => {
it(`should parse URL ${index + 1}: ${url}`, async () => {
scannedPackages = [];
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created for known pip registry");
await interceptor.handleRequest(url);
if (expected.packageName === undefined) {
assert.deepEqual(scannedPackages, []);
return;
}
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === expected.packageName &&
version === expected.version
)
);
});
});
it("should not create interceptor for unknown registry", () => {
const url = "https://example.com/packages/xx/yy/foobar-1.2.3.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.equal(interceptor, undefined);
});
it("should block malicious package", async () => {
scannedPackages = [];
const url =
"https://files.pythonhosted.org/packages/xx/yy/malicious_package-1.0.0.tar.gz";
malwareResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain"
);
malwareResponse = false;
});
});

74
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pipInterceptor.spec.js → packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipInterceptor.spec.js
View file

@ -2,22 +2,43 @@ import { describe, it, mock } from "node:test";
import assert from "node:assert";
describe("pipInterceptor", async () => {
let lastPackage;
let scannedPackages;
let malwareResponse = false;
mock.module("../../../scanning/audit/index.js", {
mock.module("../../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async (packageName, version) => {
lastPackage = { packageName, version };
scannedPackages.push({ packageName, version });
return malwareResponse;
},
},
});
mock.module("../../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: () => false,
}),
},
});
mock.module("../../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getLoggingLevel: () => "silent",
getMinimumPackageAgeHours: () => 48,
getMinimumPackageAgeExclusions: () => [],
getPipCustomRegistries: () => [],
LOGGING_SILENT: "silent",
LOGGING_VERBOSE: "verbose",
skipMinimumPackageAge: () => false,
},
});
const { pipInterceptorForUrl } = await import("./pipInterceptor.js");
const parserCases = [
// Valid pip URLs
{
url: "https://files.pythonhosted.org/packages/xx/yy/foobar-1.2.3.tar.gz",
expected: { packageName: "foobar", version: "1.2.3" },
@ -35,7 +56,6 @@ describe("pipInterceptor", async () => {
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
// Poetry preflight metadata alongside wheel (.whl.metadata)
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl.metadata",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
@ -52,7 +72,6 @@ describe("pipInterceptor", async () => {
expected: { packageName: "foo-bar", version: "2.0.0b1" },
},
{
// sdist with metadata sidecar (.tar.gz.metadata)
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0.tar.gz.metadata",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
@ -76,7 +95,6 @@ describe("pipInterceptor", async () => {
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0-cp38-cp38-manylinux1_x86_64.whl",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
// Invalid pip URLs
{
url: "https://pypi.org/simple/",
expected: { packageName: undefined, version: undefined },
@ -97,49 +115,49 @@ describe("pipInterceptor", async () => {
parserCases.forEach(({ url, expected }, index) => {
it(`should parse URL ${index + 1}: ${url}`, async () => {
scannedPackages = [];
const interceptor = pipInterceptorForUrl(url);
assert.ok(
interceptor,
"Interceptor should be created for known npm registry"
);
assert.ok(interceptor, "Interceptor should be created for known pip registry");
await interceptor.handleRequest(url);
assert.deepEqual(lastPackage, expected);
if (expected.packageName === undefined) {
assert.deepEqual(scannedPackages, []);
return;
}
assert.ok(
scannedPackages.some(
({ packageName, version }) =>
packageName === expected.packageName &&
version === expected.version
)
);
});
});
it("should not create interceptor for unknown registry", () => {
const url = "https://example.com/packages/xx/yy/foobar-1.2.3.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.equal(
interceptor,
undefined,
"Interceptor should be undefined for unknown registry"
);
assert.equal(interceptor, undefined);
});
it("should block malicious package", async () => {
scannedPackages = [];
const url =
"https://files.pythonhosted.org/packages/xx/yy/malicious_package-1.0.0.tar.gz";
malwareResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse, "Should contain a blockResponse");
assert.equal(
result.blockResponse.statusCode,
403,
"Block response should have status code 403"
);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain",
"Block response should have correct status message"
"Forbidden - blocked by safe-chain"
);
malwareResponse = false;
});
});

27
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipMetadataResponseUtils.js Normal file
View file

@ -0,0 +1,27 @@
import { getMinimumPackageAgeHours } from "../../../../config/settings.js";
import { ui } from "../../../../environment/userInteraction.js";
import { getHeaderValueAsString } from "../../http-utils.js";
import { recordSuppressedVersion } from "../suppressedVersionsState.js";
/**
* @param {NodeJS.Dict<string | string[]> | undefined} headers
* @returns {string | undefined}
*/
export function getPipMetadataContentType(headers) {
return getHeaderValueAsString(headers, "content-type")
?.toLowerCase()
.split(";")[0]
.trim();
}
/**
* @param {string} packageName
* @param {string} version
* @returns {void}
*/
export function logSuppressedVersion(packageName, version) {
recordSuppressedVersion();
ui.writeVerbose(
`Safe-chain: ${packageName}@${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`
);
}

131
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pip/pipMetadataVersionUtils.js Normal file
View file

@ -0,0 +1,131 @@
import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
/**
* @param {any} file
* @param {string} metadataUrl
* @returns {string | undefined}
*/
export function getPackageVersionFromMetadataFile(file, metadataUrl) {
const href = typeof file?.url === "string" ? file.url : undefined;
const filename = typeof file?.filename === "string" ? file.filename : undefined;
if (href) {
const resolvedHref = new URL(href, metadataUrl).toString();
return parsePipPackageFromUrl(
resolvedHref,
new URL(resolvedHref).host
).version;
}
if (filename) {
return parsePipPackageFromUrl(
new URL(filename, metadataUrl).toString(),
new URL(metadataUrl).host
).version;
}
return undefined;
}
/**
* @param {any} json
* @param {string} metadataUrl
* @returns {string[]}
*/
export function getAvailableVersionsFromJson(json, metadataUrl) {
if (json.releases && typeof json.releases === "object") {
return Object.keys(json.releases);
}
if (!Array.isArray(json.files)) {
return [];
}
return [
...new Set(
json.files
.map((/** @type {any} */ file) =>
getPackageVersionFromMetadataFile(file, metadataUrl)
)
.filter(isDefinedString)
),
];
}
/**
* @param {string | undefined} value
* @returns {value is string}
*/
function isDefinedString(value) {
return typeof value === "string";
}
/**
* @param {string[]} versions
* @returns {string | undefined}
*/
export function calculateLatestVersion(versions) {
const stableVersions = versions.filter((version) => !isPrerelease(version));
if (stableVersions.length > 0) {
return stableVersions.sort(comparePep440ishVersions).at(-1);
}
return versions.sort(comparePep440ishVersions).at(-1);
}
/**
* @param {string} left
* @param {string} right
* @returns {number}
*/
function comparePep440ishVersions(left, right) {
const leftParts = tokenizeVersion(left);
const rightParts = tokenizeVersion(right);
const maxLength = Math.max(leftParts.length, rightParts.length);
for (let index = 0; index < maxLength; index += 1) {
const leftPart = leftParts[index];
const rightPart = rightParts[index];
if (leftPart === undefined) return -1;
if (rightPart === undefined) return 1;
if (leftPart === rightPart) {
continue;
}
const leftNumeric = typeof leftPart === "number";
const rightNumeric = typeof rightPart === "number";
if (leftNumeric && rightNumeric) {
return leftPart - rightPart;
}
if (leftNumeric) return 1;
if (rightNumeric) return -1;
return String(leftPart).localeCompare(String(rightPart));
}
return 0;
}
/**
* @param {string} version
* @returns {(string | number)[]}
*/
function tokenizeVersion(version) {
return version
.toLowerCase()
.split(/[^a-z0-9]+/)
.flatMap((part) => part.match(/[a-z]+|\d+/g) || [])
.map((part) => (/^\d+$/.test(part) ? Number(part) : part));
}
/**
* @param {string} version
* @returns {boolean}
*/
function isPrerelease(version) {
return /(a|b|rc|dev)\d+/i.test(version);
}

132
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/pipInterceptor.js
View file

@ -1,132 +0,0 @@
import { getPipCustomRegistries } from "../../../config/settings.js";
import { isMalwarePackage } from "../../../scanning/audit/index.js";
import { interceptRequests } from "./interceptorBuilder.js";
const knownPipRegistries = [
"files.pythonhosted.org",
"pypi.org",
"pypi.python.org",
"pythonhosted.org",
];
/**
* @param {string} url
* @returns {import("./interceptorBuilder.js").Interceptor | undefined}
*/
export function pipInterceptorForUrl(url) {
const customRegistries = getPipCustomRegistries();
const registries = [...knownPipRegistries, ...customRegistries];
const registry = registries.find((reg) => url.includes(reg));
if (registry) {
return buildPipInterceptor(registry);
}
return undefined;
}
/**
* @param {string} registry
* @returns {import("./interceptorBuilder.js").Interceptor | undefined}
*/
function buildPipInterceptor(registry) {
return interceptRequests(async (reqContext) => {
const { packageName, version } = parsePipPackageFromUrl(
reqContext.targetUrl,
registry
);
// Normalize underscores to hyphens for DB matching, as PyPI allows underscores in distribution names.
// Per python, packages that differ only by hyphen vs underscore are considered the same.
const hyphenName = packageName?.includes("_") ? packageName.replace(/_/g, "-") : packageName;
const isMalicious =
await isMalwarePackage(packageName, version)
|| await isMalwarePackage(hyphenName, version);
if (isMalicious) {
reqContext.blockMalware(packageName, version);
}
});
}
/**
* @param {string} url
* @param {string} registry
* @returns {{packageName: string | undefined, version: string | undefined}}
*/
function parsePipPackageFromUrl(url, registry) {
let packageName, version;
// Basic validation
if (!registry || typeof url !== "string") {
return { packageName, version };
}
// Quick sanity check on the URL + parse
let urlObj;
try {
urlObj = new URL(url);
} catch {
return { packageName, version };
}
// Get the last path segment (filename) and decode it (strip query & fragment automatically)
const lastSegment = urlObj.pathname.split("/").filter(Boolean).pop();
if (!lastSegment) {
return { packageName, version };
}
const filename = decodeURIComponent(lastSegment);
// Parse Python package downloads from PyPI/pythonhosted.org
// Example wheel: https://files.pythonhosted.org/packages/xx/yy/requests-2.28.1-py3-none-any.whl
// Example sdist: https://files.pythonhosted.org/packages/xx/yy/requests-2.28.1.tar.gz
// Wheel (.whl) and Poetry's preflight metadata (.whl.metadata)
// Examples:
// foo_bar-2.0.0-py3-none-any.whl
// foo_bar-2.0.0-py3-none-any.whl.metadata
const wheelExtRe = /\.whl(?:\.metadata)?$/;
const wheelExtMatch = filename.match(wheelExtRe);
if (wheelExtMatch) {
const base = filename.replace(wheelExtRe, "");
const firstDash = base.indexOf("-");
if (firstDash > 0) {
const dist = base.slice(0, firstDash); // may contain underscores
const rest = base.slice(firstDash + 1); // version + the rest of tags
const secondDash = rest.indexOf("-");
const rawVersion = secondDash >= 0 ? rest.slice(0, secondDash) : rest;
packageName = dist;
version = rawVersion;
// Reject "latest" as it's a placeholder, not a real version
// When version is "latest", this signals the URL doesn't contain actual version info
// Returning undefined allows the request (see registryProxy.js isAllowedUrl)
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
}
// Source dist (sdist) and potential metadata sidecars (e.g., .tar.gz.metadata)
const sdistExtWithMetadataRe = /\.(tar\.gz|zip|tar\.bz2|tar\.xz)(\.metadata)?$/i;
const sdistExtMatch = filename.match(sdistExtWithMetadataRe);
if (sdistExtMatch) {
const base = filename.replace(sdistExtWithMetadataRe, "");
const lastDash = base.lastIndexOf("-");
if (lastDash > 0 && lastDash < base.length - 1) {
packageName = base.slice(0, lastDash);
version = base.slice(lastDash + 1);
// Reject "latest" as it's a placeholder, not a real version
// When version is "latest", this signals the URL doesn't contain actual version info
// Returning undefined allows the request (see registryProxy.js isAllowedUrl)
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
}
// Unknown file type or invalid
return { packageName: undefined, version: undefined };
}

21
packages/safe-chain/src/registryProxy/builtInProxy/interceptors/suppressedVersionsState.js Normal file
View file

@ -0,0 +1,21 @@
const state = {
hasSuppressedVersions: false,
};
/**
* Tracks whether any rewritten metadata response suppressed versions during the
* current process lifetime. This is intentional shared state used only for the
* end-of-run summary message exposed through the proxy API.
*
* @returns {void}
*/
export function recordSuppressedVersion() {
state.hasSuppressedVersions = true;
}
/**
* @returns {boolean}
*/
export function getHasSuppressedVersions() {
return state.hasSuppressedVersions;
}

18
packages/safe-chain/src/registryProxy/builtInProxy/mitmRequestHandler.js
View file

@ -2,7 +2,8 @@ import https from "https";
import { generateCertForHost } from "./certUtils.js";
import { HttpsProxyAgent } from "https-proxy-agent";
import { ui } from "../../environment/userInteraction.js";
import { gunzipSync, gzipSync } from "zlib";
import { gunzipSync } from "zlib";
import { omitHeaders } from "./http-utils.js";
/**
* @typedef {import("./interceptors/interceptorBuilder.js").Interceptor} Interceptor
@ -215,11 +216,16 @@ function createProxyRequest(hostname, port, req, res, requestHandler) {
buffer = requestHandler.modifyBody(buffer, headers);
if (proxyRes.headers["content-encoding"] === "gzip") {
buffer = gzipSync(buffer);
}
res.writeHead(statusCode, headers);
// For rewritten responses, send the final body uncompressed.
// This avoids mismatches between upstream compression metadata and the
// rewritten payload on the wire.
const rewrittenHeaders = omitHeaders(
headers,
["content-length", "transfer-encoding", "content-encoding"],
{ caseInsensitive: true }
) || {};
rewrittenHeaders["content-length"] = String(buffer.byteLength);
res.writeHead(statusCode, rewrittenHeaders);
res.end(buffer);
});
} else {

138
packages/safe-chain/src/registryProxy/builtInProxy/mitmRequestHandler.spec.js Normal file
View file

@ -0,0 +1,138 @@
import { describe, it, mock } from "node:test";
import assert from "node:assert";
import zlib from "node:zlib";
describe("mitmRequestHandler", async () => {
let capturedHandler;
let capturedOptions;
mock.module("https", {
defaultExport: {
createServer: (_options, handler) => {
capturedHandler = handler;
return {
on: () => {},
emit: () => {},
};
},
request: (options, callback) => {
capturedOptions = options;
const listeners = {};
const proxyRes = {
statusCode: 200,
headers: {
"content-encoding": "gzip",
"content-length": "999",
"transfer-encoding": "chunked",
},
on: (event, handler) => {
listeners[event] = handler;
},
};
callback(proxyRes);
return {
on: () => {},
write: () => {},
end: () => {
const payload = Buffer.from("rewritten body");
listeners["data"]?.(zlib.gzipSync(payload));
listeners["end"]?.();
},
destroy: () => {},
};
},
},
});
mock.module("./certUtils.js", {
namedExports: {
generateCertForHost: () => ({
privateKey: "key",
certificate: "cert",
}),
},
});
mock.module("https-proxy-agent", {
namedExports: {
HttpsProxyAgent: class {},
},
});
mock.module("../../environment/userInteraction.js", {
namedExports: {
ui: {
writeVerbose: () => {},
writeError: () => {},
},
},
});
const { mitmConnect } = await import("./mitmRequestHandler.js");
it("sets content-length from the final compressed payload after body rewrite", async () => {
const interceptor = {
handleRequest: async () => ({
blockResponse: undefined,
modifyRequestHeaders: (headers) => headers,
modifiesResponse: () => true,
modifyBody: () => Buffer.from("rewritten body"),
}),
};
const req = {
url: "pypi.org:443",
};
const clientSocket = {
on: () => {},
write: () => {},
headersSent: false,
writable: true,
end: () => {},
};
mitmConnect(req, clientSocket, interceptor);
const resState = {
statusCode: undefined,
headers: undefined,
body: undefined,
};
const res = {
headersSent: false,
writeHead: (statusCode, headers) => {
resState.statusCode = statusCode;
resState.headers = headers;
},
end: (body) => {
resState.body = body;
},
};
const request = {
url: "/simple/example/",
headers: {},
method: "GET",
on: (event, handler) => {
if (event === "end") {
handler();
}
},
};
await capturedHandler(request, res);
assert.equal(capturedOptions.hostname, "pypi.org");
assert.equal(resState.statusCode, 200);
assert.equal(resState.headers["transfer-encoding"], undefined);
assert.equal(
resState.headers["content-length"],
String(resState.body.byteLength)
);
});
});

28
packages/safe-chain/src/registryProxy/certBundle.js
View file

@ -7,6 +7,9 @@ import tls from "node:tls";
import { X509Certificate } from "node:crypto";
import { ui } from "../environment/userInteraction.js";
/** @type {string | null} */
let bundlePath = null;
/**
* Check if a PEM string contains only parsable cert blocks.
* @param {string} pem - PEM-encoded certificate string
@ -54,6 +57,11 @@ function isParsable(pem) {
* @returns {string} Path to the combined CA bundle PEM file
*/
export function getCombinedCaBundlePath(proxyCaCert) {
if (bundlePath)
{
return bundlePath;
}
// 1) Safe Chain CA (for MITM'd registries)
const parts = [];
if (proxyCaCert && isParsable(proxyCaCert)) parts.push(proxyCaCert.trim());
@ -92,9 +100,23 @@ export function getCombinedCaBundlePath(proxyCaCert) {
}
const combined = parts.filter(Boolean).join("\n");
const target = path.join(os.tmpdir(), `safe-chain-ca-bundle-${Date.now()}.pem`);
fs.writeFileSync(target, combined, { encoding: "utf8" });
return target;
bundlePath = path.join(os.tmpdir(), `safe-chain-ca-bundle-${Date.now()}.pem`);
fs.writeFileSync(bundlePath, combined, { encoding: "utf8" });
return bundlePath;
}
/**
* Remove the generated CA bundle file from disk.
*/
export function cleanupCertBundle() {
if (bundlePath) {
try {
fs.unlinkSync(bundlePath);
} catch (err) {
ui.writeVerbose(`Failed to cleanup the create bundle at ${bundlePath}`, err)
}
bundlePath = null;
}
}
/**

8
packages/safe-chain/src/registryProxy/registryProxy.js
View file

@ -4,12 +4,12 @@ import { createBuiltInProxyServer } from "./builtInProxy/createBuiltInProxyServe
import { getCombinedCaBundlePath } from "./certBundle.js";
/**
* @typedef {Object} MalwareBlockedEvent
* @typedef {Object} PackageBlockedEvent
* @prop {string} packageName
* @prop {string} packageVersion
*
* @typedef {{ malwareBlocked: [MalwareBlockedEvent] }} ProxyServerEvents
*
*
* @typedef {{ malwareBlocked: [PackageBlockedEvent], minimumAgeRequestBlocked: [PackageBlockedEvent] }} ProxyServerEvents
*
* @import { EventEmitter } from "node:stream"
* @typedef {EventEmitter<ProxyServerEvents> & {
* startServer: () => Promise<void>