Merge pull request #332 from AikidoSec/rama-min-package-age-reporting

Rama min package age reporting
This commit is contained in:
bitterpanda 2026-05-06 14:56:33 +08:00 committed by GitHub
commit eedbac7e28
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 188 additions and 101 deletions

View file

@ -24,10 +24,15 @@ export async function main(args) {
let malwareBlockedEvents = [];
/** @type {import("./registryProxy/registryProxy.js").PackageBlockedEvent[]} */
let minPackageAgeBlocks = [];
/** @type {import("./registryProxy/registryProxy.js").MinPackageAgeSuppressionEvent[]} */
let suppressedVersionEvents = [];
const proxy = createSafeChainProxy();
await proxy.startServer();
proxy.addListener("malwareBlocked", (ev) => malwareBlockedEvents.push(ev));
proxy.addListener("minPackageAgeVersionsSuppressed", (ev) =>
suppressedVersionEvents.push(ev),
);
proxy.addListener("minimumAgeRequestBlocked", (ev) =>
minPackageAgeBlocks.push(ev),
);
@ -92,17 +97,8 @@ export async function main(args) {
);
}
if (proxy.hasSuppressedVersions()) {
ui.writeInformation(
`${chalk.yellow(
"",
)} Safe-chain: Some package versions were suppressed during package metadata resolution due to minimum package age.`,
);
ui.writeInformation(
` To disable this check, use: ${chalk.cyan(
"--safe-chain-skip-minimum-package-age",
)}`,
);
if (suppressedVersionEvents.length > 0) {
printSuppressedVersions(suppressedVersionEvents);
}
// Returning the exit code back to the caller allows the promise
@ -184,3 +180,25 @@ function printMinPackageAgeBlocks(minPackageAgeBlocks) {
);
ui.emptyLine();
}
/**
*
* @param {import("./registryProxy/registryProxy.js").MinPackageAgeSuppressionEvent[]} minPackageAgeSuppressionEvents
*/
function printSuppressedVersions(minPackageAgeSuppressionEvents) {
ui.writeVerbose(
`${chalk.yellow(
"",
)} Safe-chain: Some package versions were suppressed during package metadata resolution due to minimum package age:`,
);
for (const ev of minPackageAgeSuppressionEvents) {
ui.writeVerbose(` - ${ev.packageName} (${ev.packageVersions.join(", ")})`);
}
ui.writeVerbose(
` To disable this check, use: ${chalk.cyan(
"--safe-chain-skip-minimum-package-age",
)}`,
);
}

View file

@ -7,8 +7,9 @@ import { createInterceptorForUrl } from "./interceptors/createInterceptorForEcoS
import { getCaCertPath } from "./certUtils.js";
import { readFileSync } from "fs";
import EventEmitter from "events";
import { modifyResponseEventEmitter } from "./interceptors/npm/modifyNpmInfo.js";
import { modifyPipResponseEventEmitter } from "./interceptors/pip/modifyPipInfo.js";
import { cleanupCertBundle } from "../certBundle.js";
import { getHasSuppressedVersions } from "./interceptors/suppressedVersionsState.js";
/** *
* @returns {import("../registryProxy.js").SafeChainProxy} */
@ -23,6 +24,14 @@ export function createBuiltInProxyServer() {
/** @type {EventEmitter<import("../registryProxy.js").ProxyServerEvents>} */
const emitter = new EventEmitter();
modifyResponseEventEmitter.addListener("versionsRemoved", (ev) => {
emitter.emit("minPackageAgeVersionsSuppressed", ev);
});
modifyPipResponseEventEmitter.addListener("versionsRemoved", (ev) => {
emitter.emit("minPackageAgeVersionsSuppressed", ev);
});
const server = http.createServer(
// This handles direct HTTP requests (non-CONNECT requests)
// This is normally http-only traffic, but we also handle
@ -36,7 +45,6 @@ export function createBuiltInProxyServer() {
return Object.assign(emitter, {
startServer: () => startServer(server),
stopServer: () => stopServer(server),
hasSuppressedVersions: getHasSuppressedVersions,
getServerPort: () => state.port,
getCaCert,
});

View file

@ -7,7 +7,7 @@ const mockMitmConnect = mock.fn();
const mockTunnelRequest = mock.fn();
const mockUi = { writeVerbose: mock.fn() };
const mockGetCaCertPath = mock.fn(() => "/fake/cert/path");
const mockGetHasSuppressedVersions = mock.fn(() => false);
const mockModifyResponseEventEmitter = new EventEmitter();
/** @type {import("./interceptors/interceptorBuilder.js").Interceptor | undefined} */
let mockInterceptor;
@ -30,7 +30,7 @@ mock.module("./interceptors/createInterceptorForEcoSystem.js", {
},
});
mock.module("./interceptors/npm/modifyNpmInfo.js", {
namedExports: { getHasSuppressedVersions: mockGetHasSuppressedVersions },
namedExports: { modifyResponseEventEmitter: mockModifyResponseEventEmitter },
});
mock.module("./certUtils.js", {
namedExports: { getCaCertPath: mockGetCaCertPath },

View file

@ -1,7 +1,13 @@
import { EventEmitter } from "events";
import { getMinimumPackageAgeHours } from "../../../../config/settings.js";
import { ui } from "../../../../environment/userInteraction.js";
import { clearCachingHeaders, getHeaderValueAsString } from "../../http-utils.js";
import { recordSuppressedVersion } from "../suppressedVersionsState.js";
import {
clearCachingHeaders,
getHeaderValueAsString,
} from "../../http-utils.js";
/** @type {EventEmitter<{ versionsRemoved: [{packageName: string, packageVersions: string[]}] }>} */
export const modifyResponseEventEmitter = new EventEmitter();
/**
* @param {NodeJS.Dict<string | string[]>} headers
@ -62,8 +68,10 @@ export function modifyNpmInfoResponse(body, headers) {
return body;
}
const packageName = bodyJson.name;
const cutOff = new Date(
new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000,
);
const hasLatestTag = !!bodyJson["dist-tags"]["latest"];
@ -75,9 +83,13 @@ export function modifyNpmInfoResponse(body, headers) {
}))
.filter((x) => x.version !== "created" && x.version !== "modified");
const removedVersions = [];
for (const { version, timestamp } of versions) {
const timestampValue = new Date(timestamp);
if (timestampValue > cutOff) {
removedVersions.push(version);
deleteVersionFromJson(bodyJson, version);
clearCachingHeaders(headers);
}
@ -89,10 +101,17 @@ export function modifyNpmInfoResponse(body, headers) {
bodyJson["dist-tags"]["latest"] = calculateLatestTag(bodyJson.time);
}
if (removedVersions.length > 0) {
modifyResponseEventEmitter.emit("versionsRemoved", {
packageName: packageName,
packageVersions: removedVersions,
});
}
return Buffer.from(JSON.stringify(bodyJson));
} catch (/** @type {any} */ err) {
ui.writeVerbose(
`Safe-chain: Package metadata not in expected format - bypassing modification. Error: ${err.message}`
`Safe-chain: Package metadata not in expected format - bypassing modification. Error: ${err.message}`,
);
return body;
}
@ -103,12 +122,10 @@ export function modifyNpmInfoResponse(body, headers) {
* @param {string} version
*/
function deleteVersionFromJson(json, version) {
recordSuppressedVersion();
const packageName = typeof json?.name === "string" ? json.name : "(unknown)";
ui.writeVerbose(
`Safe-chain: ${packageName}@${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`
`Safe-chain: ${packageName}@${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`,
);
delete json.time[version];
@ -127,18 +144,20 @@ function deleteVersionFromJson(json, version) {
*/
function calculateLatestTag(tagList) {
const entries = Object.entries(tagList).filter(
([version, _]) => version !== "created" && version !== "modified"
([version, _]) => version !== "created" && version !== "modified",
);
const latestFullRelease = getMostRecentTag(
Object.fromEntries(entries.filter(([version, _]) => !version.includes("-")))
Object.fromEntries(
entries.filter(([version, _]) => !version.includes("-")),
),
);
if (latestFullRelease) {
return latestFullRelease;
}
const latestPrerelease = getMostRecentTag(
Object.fromEntries(entries.filter(([version, _]) => version.includes("-")))
Object.fromEntries(entries.filter(([version, _]) => version.includes("-"))),
);
return latestPrerelease;
}

View file

@ -1,3 +1,4 @@
import { EventEmitter } from "events";
import { ui } from "../../../../environment/userInteraction.js";
import { clearCachingHeaders } from "../../http-utils.js";
import { normalizePipPackageName } from "../../../../scanning/packageNameVariants.js";
@ -6,6 +7,9 @@ export { parsePipMetadataUrl, isPipPackageInfoUrl } from "./parsePipPackageUrl.j
import { getPipMetadataContentType, logSuppressedVersion } from "./pipMetadataResponseUtils.js";
import { modifyPipJsonResponse } from "./modifyPipJsonResponse.js";
/** @type {EventEmitter<{ versionsRemoved: [{packageName: string, packageVersions: string[]}] }>} */
export const modifyPipResponseEventEmitter = new EventEmitter();
/**
* Strip conditional GET headers so PyPI always returns a full 200 response
* with a body we can rewrite. Without this, pip sends If-None-Match /
@ -50,33 +54,42 @@ export function modifyPipInfoResponse(
return body;
}
/** @type {{ buffer: Buffer, suppressedVersions: string[] } | undefined} */
let result;
if (
contentType.includes("html") ||
contentType.includes("application/vnd.pypi.simple.v1+html")
) {
return modifyHtmlSimpleResponse(
result = modifyHtmlSimpleResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
}
if (
} else if (
contentType.includes("json") ||
contentType.includes("application/vnd.pypi.simple.v1+json")
) {
return modifyJsonResponse(
result = modifyJsonResponse(
body,
headers,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
} else {
return body;
}
return body;
if (result.suppressedVersions.length > 0) {
modifyPipResponseEventEmitter.emit("versionsRemoved", {
packageName,
packageVersions: result.suppressedVersions,
});
}
return result.buffer;
} catch (/** @type {any} */ err) {
ui.writeVerbose(
`Safe-chain: PyPI package metadata not in expected format - bypassing modification. Error: ${err.message}`
@ -91,7 +104,7 @@ export function modifyPipInfoResponse(
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {Buffer}
* @returns {{ buffer: Buffer, suppressedVersions: string[] }}
*/
function modifyHtmlSimpleResponse(
body,
@ -101,35 +114,35 @@ function modifyHtmlSimpleResponse(
packageName
) {
const html = body.toString("utf8");
let modified = false;
const suppressedVersions = /** @type {string[]} */ ([]);
const rewriteHtmlAnchor = createHtmlAnchorRewriter(
metadataUrl,
isNewlyReleasedPackage,
packageName,
() => {
modified = true;
(version) => {
suppressedVersions.push(version);
}
);
const updatedHtml = html.replace(HTML_ANCHOR_HREF_RE, rewriteHtmlAnchor);
if (!modified) return body;
if (suppressedVersions.length === 0) return { buffer: body, suppressedVersions: [] };
const modifiedBuffer = Buffer.from(updatedHtml);
clearCachingHeaders(headers);
return modifiedBuffer;
return { buffer: modifiedBuffer, suppressedVersions: [...new Set(suppressedVersions)] };
}
/**
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @param {() => void} onModified
* @param {(version: string) => void} onVersionSuppressed
* @returns {(anchor: string, quote: string, href: string) => string}
*/
function createHtmlAnchorRewriter(
metadataUrl,
isNewlyReleasedPackage,
packageName,
onModified
onVersionSuppressed
) {
return (anchor, _quote, href) => {
const resolvedHref = new URL(href, metadataUrl).toString();
@ -145,8 +158,8 @@ function createHtmlAnchorRewriter(
version &&
isNewlyReleasedPackage(packageName, version)
) {
onModified();
logSuppressedVersion(packageName, version);
onVersionSuppressed(version);
return "";
}
@ -160,7 +173,7 @@ function createHtmlAnchorRewriter(
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {Buffer}
* @returns {{ buffer: Buffer, suppressedVersions: string[] }}
*/
function modifyJsonResponse(
body,
@ -170,15 +183,15 @@ function modifyJsonResponse(
packageName
) {
const json = JSON.parse(body.toString("utf8"));
const modified = modifyPipJsonResponse(
const { suppressedVersions, wasModified } = modifyPipJsonResponse(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
if (!modified) return body;
if (!wasModified) return { buffer: body, suppressedVersions: [] };
const modifiedBuffer = Buffer.from(JSON.stringify(json));
clearCachingHeaders(headers);
return modifiedBuffer;
return { buffer: modifiedBuffer, suppressedVersions };
}

View file

@ -10,7 +10,7 @@ import { logSuppressedVersion } from "./pipMetadataResponseUtils.js";
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
* @returns {{ suppressedVersions: string[], wasModified: boolean }}
*/
export function modifyPipJsonResponse(
json,
@ -18,18 +18,18 @@ export function modifyPipJsonResponse(
isNewlyReleasedPackage,
packageName
) {
const filesModified = filterJsonMetadataFiles(
const filesSuppressed = filterJsonMetadataFiles(
json,
metadataUrl,
isNewlyReleasedPackage,
packageName
);
const releasesModified = removeJsonMetadataReleases(
const releasesSuppressed = removeJsonMetadataReleases(
json,
isNewlyReleasedPackage,
packageName
);
const urlsModified = filterJsonMetadataUrls(
const urlsSuppressed = filterJsonMetadataUrls(
json,
metadataUrl,
isNewlyReleasedPackage,
@ -37,7 +37,11 @@ export function modifyPipJsonResponse(
);
const versionModified = updateJsonInfoVersion(json, metadataUrl);
return filesModified || releasesModified || urlsModified || versionModified;
const suppressedVersions = [
...new Set([...filesSuppressed, ...releasesSuppressed, ...urlsSuppressed]),
];
return { suppressedVersions, wasModified: suppressedVersions.length > 0 || versionModified };
}
/**
@ -45,7 +49,7 @@ export function modifyPipJsonResponse(
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
* @returns {string[]}
*/
function filterJsonMetadataFiles(
json,
@ -54,19 +58,17 @@ function filterJsonMetadataFiles(
packageName
) {
if (!Array.isArray(json.files)) {
return false;
return [];
}
let modified = false;
const loggedVersions = new Set();
const suppressed = new Set();
json.files = json.files.filter((/** @type {any} */ file) => {
const version = getPackageVersionFromMetadataFile(file, metadataUrl);
if (version && isNewlyReleasedPackage(packageName, version)) {
modified = true;
if (!loggedVersions.has(version)) {
if (!suppressed.has(version)) {
logSuppressedVersion(packageName, version);
loggedVersions.add(version);
suppressed.add(version);
}
return false;
}
@ -74,21 +76,21 @@ function filterJsonMetadataFiles(
return true;
});
return modified;
return [...suppressed];
}
/**
* @param {any} json
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
* @returns {string[]}
*/
function removeJsonMetadataReleases(json, isNewlyReleasedPackage, packageName) {
if (!json.releases || typeof json.releases !== "object") {
return false;
return [];
}
let modified = false;
const suppressed = [];
for (const [version, files] of Object.entries(json.releases)) {
if (
@ -96,12 +98,12 @@ function removeJsonMetadataReleases(json, isNewlyReleasedPackage, packageName) {
isNewlyReleasedPackage(packageName, version)
) {
delete json.releases[version];
modified = true;
logSuppressedVersion(packageName, version);
suppressed.push(version);
}
}
return modified;
return suppressed;
}
/**
@ -109,7 +111,7 @@ function removeJsonMetadataReleases(json, isNewlyReleasedPackage, packageName) {
* @param {string} metadataUrl
* @param {(packageName: string | undefined, version: string | undefined) => boolean} isNewlyReleasedPackage
* @param {string} packageName
* @returns {boolean}
* @returns {string[]}
*/
function filterJsonMetadataUrls(
json,
@ -118,19 +120,17 @@ function filterJsonMetadataUrls(
packageName
) {
if (!Array.isArray(json.urls)) {
return false;
return [];
}
let modified = false;
const loggedVersions = new Set();
const suppressed = new Set();
json.urls = json.urls.filter((/** @type {any} */ file) => {
const version = getPackageVersionFromMetadataFile(file, metadataUrl);
if (version && isNewlyReleasedPackage(packageName, version)) {
modified = true;
if (!loggedVersions.has(version)) {
if (!suppressed.has(version)) {
logSuppressedVersion(packageName, version);
loggedVersions.add(version);
suppressed.add(version);
}
return false;
}
@ -138,7 +138,7 @@ function filterJsonMetadataUrls(
return true;
});
return modified;
return [...suppressed];
}
/**

View file

@ -1,7 +1,6 @@
import { getMinimumPackageAgeHours } from "../../../../config/settings.js";
import { ui } from "../../../../environment/userInteraction.js";
import { getHeaderValueAsString } from "../../http-utils.js";
import { recordSuppressedVersion } from "../suppressedVersionsState.js";
/**
* @param {NodeJS.Dict<string | string[]> | undefined} headers
@ -20,7 +19,6 @@ export function getPipMetadataContentType(headers) {
* @returns {void}
*/
export function logSuppressedVersion(packageName, version) {
recordSuppressedVersion();
ui.writeVerbose(
`Safe-chain: ${packageName}@${version} is newer than ${getMinimumPackageAgeHours()} hours and was removed (minimumPackageAgeInHours setting).`
);

View file

@ -1,21 +0,0 @@
const state = {
hasSuppressedVersions: false,
};
/**
* Tracks whether any rewritten metadata response suppressed versions during the
* current process lifetime. This is intentional shared state used only for the
* end-of-run summary message exposed through the proxy API.
*
* @returns {void}
*/
export function recordSuppressedVersion() {
state.hasSuppressedVersions = true;
}
/**
* @returns {boolean}
*/
export function getHasSuppressedVersions() {
return state.hasSuppressedVersions;
}

View file

@ -48,10 +48,24 @@ export function createRamaProxy(ramaPath) {
return Object.assign(emitter, {
startServer: async () => {
await reportingServer.start();
reportingServer.addListener("blockReceived", (ev) =>
emitter.emit("malwareBlocked", {
reportingServer.addListener("blockReceived", (ev) => {
if (ev.block_reason === "new_package") {
emitter.emit("minimumAgeRequestBlocked", {
packageName: ev.artifact.identifier,
packageVersion: ev.artifact.version,
});
}
else {
emitter.emit("malwareBlocked", {
packageName: ev.artifact.identifier,
packageVersion: ev.artifact.version,
});
}
});
reportingServer.addListener("minPackageAgeSuppressionReceived", (ev) =>
emitter.emit("minPackageAgeVersionsSuppressed", {
packageName: ev.artifact.identifier,
packageVersion: ev.artifact.version,
packageVersions: ev.suppressed_versions,
}),
);
ui.writeVerbose(

View file

@ -7,10 +7,18 @@ const SERVER_STOP_TIMEOUT_MS = 1000;
* @typedef {Object} BlockEvent
* @property {number} ts_ms
* @property {{ product: string, identifier: string, version: string }} artifact
* @property {string} block_reason
*/
/**
* @typedef {{ blockReceived: [BlockEvent] }} ReportingServerEvents
* @typedef {Object} MinPackageAgeEvent
* @property {number} ts_ms
* @property {{ product: string, identifier: string }} artifact
* @property {string[]} suppressed_versions
*/
/**
* @typedef {{ blockReceived: [BlockEvent], minPackageAgeSuppressionReceived: [MinPackageAgeEvent] }} ReportingServerEvents
*/
/**
@ -38,6 +46,11 @@ export function getReportingServer() {
emitter.emit("blockReceived", blockEvent);
});
}
else if (req.method === "POST" && req.url?.startsWith("/events/min-package-age")) {
await parseMinPackageAgeEventFromRequest(req).then((minPackageAgeEvent) => {
emitter.emit("minPackageAgeSuppressionReceived", minPackageAgeEvent);
});
}
res.writeHead(200);
res.end();
}
@ -75,12 +88,30 @@ export function getReportingServer() {
* @param {http.IncomingMessage} req
* @returns {Promise<BlockEvent>}
*/
function parseBlockEventFromRequest(req) {
async function parseBlockEventFromRequest(req) {
const requestData = await getRequestDataAsString(req);
return JSON.parse(requestData);
}
/**
* @param {http.IncomingMessage} req
* @returns {Promise<MinPackageAgeEvent>}
*/
async function parseMinPackageAgeEventFromRequest(req) {
const requestData = await getRequestDataAsString(req);
return JSON.parse(requestData);
}
/**
* @param {http.IncomingMessage} req
* @returns {Promise<string>}
*/
function getRequestDataAsString(req) {
return new Promise((resolve, reject) => {
/** @type {Buffer[]} */
const chunks = [];
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => resolve(JSON.parse(Buffer.concat(chunks).toString())));
req.on("end", () => resolve(Buffer.concat(chunks).toString()));
req.on("error", reject);
});
}

View file

@ -8,7 +8,15 @@ import { getCombinedCaBundlePath } from "./certBundle.js";
* @prop {string} packageName
* @prop {string} packageVersion
*
* @typedef {{ malwareBlocked: [PackageBlockedEvent], minimumAgeRequestBlocked: [PackageBlockedEvent] }} ProxyServerEvents
* @typedef {Object} MinPackageAgeSuppressionEvent
* @prop {string} packageName
* @prop {string[]} packageVersions
*
* @typedef {{
* malwareBlocked: [PackageBlockedEvent],
* minimumAgeRequestBlocked: [PackageBlockedEvent]
* minPackageAgeVersionsSuppressed: [MinPackageAgeSuppressionEvent]
* }} ProxyServerEvents
*
* @import { EventEmitter } from "node:stream"
* @typedef {EventEmitter<ProxyServerEvents> & {
@ -16,7 +24,6 @@ import { getCombinedCaBundlePath } from "./certBundle.js";
* stopServer: () => Promise<void>
* getServerPort: () => Number | null
* getCaCert: () => string | null
* hasSuppressedVersions: () => boolean
* }} SafeChainProxy
*
* @typedef {Object} ProxySettings