Add minimum package age check for pypi

This commit is contained in:
Reinier Criel 2026-03-28 10:15:13 -07:00
parent 2c8a1b4972
commit fd6fb456b4
22 changed files with 516 additions and 273 deletions

View file

@ -111,17 +111,20 @@ safe-chain --version
The Aikido Safe Chain works by running a lightweight proxy server that intercepts package downloads from the npm registry and PyPI. When you run npm, npx, yarn, pnpm, pnpx, bun, bunx, pip, pip3, uv, poetry or pipx commands, all package downloads are routed through this local proxy, which verifies packages in real-time against **[Aikido Intel - Open Sources Threat Intelligence](https://intel.aikido.dev/?tab=malware)**. If malware is detected in any package (including deep dependencies), the proxy blocks the download before the malicious code reaches your machine.
### Minimum package age (npm only)
### Minimum package age
For npm packages, Safe Chain applies minimum package age checks in two ways:
Safe Chain applies minimum package age checks to supported ecosystems.
- During normal package resolution, Safe Chain suppresses versions that are newer than the configured minimum age from the package metadata returned by the registry.
- For direct package download requests that bypass that metadata flow, Safe Chain can block the request itself using a cached list of newly released packages.
Current enforcement differs by ecosystem:
- npm-based package managers:
- during normal package resolution, Safe Chain suppresses versions that are newer than the configured minimum age from the package metadata returned by the registry
- for direct package download requests that bypass that metadata flow, Safe Chain can block the request itself using a cached list of newly released packages
- Python package managers:
- Safe Chain blocks direct package download requests using a cached list of newly released packages
By default, the minimum package age is 48 hours. This provides an additional security layer during the critical period when newly published packages are most vulnerable to containing undetected threats. You can configure this threshold or bypass this protection entirely - see the [Minimum Package Age Configuration](#minimum-package-age) section below.
⚠️ This feature **only applies to npm-based package managers** (npm, npx, yarn, pnpm, pnpx, bun, bunx) and does not apply to Python package managers (uv, pip, pip3, poetry, pipx).
### Shell Integration
The Aikido Safe Chain integrates with your shell to provide a seamless experience when using npm, npx, yarn, pnpm, pnpx, bun, bunx, and Python package managers (pip, uv, poetry, pipx). It sets up aliases for these commands so that they are wrapped by the Aikido Safe Chain commands, which manage the proxy server before executing the original commands. We currently support:
@ -188,13 +191,15 @@ You can set the logging level through multiple sources (in order of priority):
## Minimum Package Age
You can configure how long packages must exist before Safe Chain allows their installation. By default, packages must be at least 48 hours old before they can be installed through npm-based package managers.
You can configure how long packages must exist before Safe Chain allows their installation. By default, packages must be at least 48 hours old before they can be installed.
For npm-based package managers, this check currently has two enforcement modes:
- Safe Chain suppresses too-young versions from package metadata during normal dependency resolution.
- Safe Chain blocks direct package download requests when they are matched against the cached newly released packages list.
For Python package managers, Safe Chain currently enforces minimum package age by blocking direct package download requests when they are matched against the cached newly released packages list.
### Configuration Options
You can set the minimum package age through multiple sources (in order of priority):
@ -225,13 +230,16 @@ You can set the minimum package age through multiple sources (in order of priori
Exclude trusted packages from minimum age filtering via environment variable or config file (both are merged). Use `@scope/*` to trust all packages from an organization:
```shell
export SAFE_CHAIN_NPM_MINIMUM_PACKAGE_AGE_EXCLUSIONS="@aikidosec/*"
export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="@aikidosec/*"
```
```json
{
"npm": {
"minimumPackageAgeExclusions": ["@aikidosec/*"]
},
"pip": {
"minimumPackageAgeExclusions": ["requests"]
}
}
```

View file

@ -129,18 +129,21 @@ export function getPipCustomRegistries() {
}
/**
* Gets the minimum package age exclusions from the config file
* Gets the minimum package age exclusions from the config file for the current ecosystem
* @returns {string[]}
*/
export function getNpmMinimumPackageAgeExclusions() {
export function getMinimumPackageAgeExclusions() {
const config = readConfigFile();
const ecosystem = getEcoSystem();
const registryConfig = ecosystem === "py" ? config.pip : config.npm;
if (!config || !config.npm) {
if (!config || !registryConfig) {
return [];
}
const npmConfig = /** @type {SafeChainRegistryConfiguration} */ (config.npm);
const exclusions = npmConfig.minimumPackageAgeExclusions;
const typedRegistryConfig =
/** @type {SafeChainRegistryConfiguration} */ (registryConfig);
const exclusions = typedRegistryConfig.minimumPackageAgeExclusions;
if (!Array.isArray(exclusions)) {
return [];

View file

@ -41,6 +41,7 @@ export function getLoggingLevel() {
* Example: "react,@aikidosec/safe-chain,lodash"
* @returns {string | undefined}
*/
export function getNpmMinimumPackageAgeExclusions() {
return process.env.SAFE_CHAIN_NPM_MINIMUM_PACKAGE_AGE_EXCLUSIONS;
export function getMinimumPackageAgeExclusions() {
return process.env.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS ||
process.env.SAFE_CHAIN_NPM_MINIMUM_PACKAGE_AGE_EXCLUSIONS;
}

View file

@ -188,11 +188,11 @@ function parseExclusionsFromEnv(envValue) {
* Gets the minimum package age exclusions from both environment variable and config file (merged)
* @returns {string[]}
*/
export function getNpmMinimumPackageAgeExclusions() {
export function getMinimumPackageAgeExclusions() {
const envExclusions = parseExclusionsFromEnv(
environmentVariables.getNpmMinimumPackageAgeExclusions()
environmentVariables.getMinimumPackageAgeExclusions()
);
const configExclusions = configFile.getNpmMinimumPackageAgeExclusions();
const configExclusions = configFile.getMinimumPackageAgeExclusions();
// Merge both sources and remove duplicates
const allExclusions = [...envExclusions, ...configExclusions];

View file

@ -14,7 +14,10 @@ mock.module("fs", {
const {
getNpmCustomRegistries,
getPipCustomRegistries,
getNpmMinimumPackageAgeExclusions,
getMinimumPackageAgeExclusions,
setEcoSystem,
ECOSYSTEM_JS,
ECOSYSTEM_PY,
getLoggingLevel,
LOGGING_SILENT,
LOGGING_NORMAL,
@ -367,13 +370,18 @@ describe("getLoggingLevel", () => {
});
});
describe("getNpmMinimumPackageAgeExclusions", () => {
describe("getMinimumPackageAgeExclusions", () => {
let originalEnv;
const envVarName = "SAFE_CHAIN_NPM_MINIMUM_PACKAGE_AGE_EXCLUSIONS";
let originalLegacyEnv;
const envVarName = "SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS";
const legacyEnvVarName = "SAFE_CHAIN_NPM_MINIMUM_PACKAGE_AGE_EXCLUSIONS";
beforeEach(() => {
originalEnv = process.env[envVarName];
originalLegacyEnv = process.env[legacyEnvVarName];
delete process.env[envVarName];
delete process.env[legacyEnvVarName];
setEcoSystem(ECOSYSTEM_JS);
});
afterEach(() => {
@ -382,13 +390,18 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
} else {
delete process.env[envVarName];
}
if (originalLegacyEnv !== undefined) {
process.env[legacyEnvVarName] = originalLegacyEnv;
} else {
delete process.env[legacyEnvVarName];
}
configFileContent = undefined;
});
it("should return empty array when no exclusions configured", () => {
configFileContent = undefined;
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, []);
});
@ -400,7 +413,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
},
});
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["react", "@aikidosec/safe-chain"]);
});
@ -409,7 +422,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
process.env[envVarName] = "lodash,express,@types/node";
configFileContent = undefined;
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["lodash", "express", "@types/node"]);
});
@ -422,7 +435,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
},
});
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["lodash", "react"]);
});
@ -435,7 +448,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
},
});
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["lodash", "react", "express"]);
});
@ -444,7 +457,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
process.env[envVarName] = " lodash , react ";
configFileContent = undefined;
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["lodash", "react"]);
});
@ -456,7 +469,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
},
});
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["@babel/core", "@types/react"]);
});
@ -465,7 +478,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
process.env[envVarName] = "lodash,,react,";
configFileContent = undefined;
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["lodash", "react"]);
});
@ -474,7 +487,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
process.env[envVarName] = "";
configFileContent = undefined;
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, []);
});
@ -483,7 +496,7 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
process.env[envVarName] = " , , ";
configFileContent = undefined;
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, []);
});
@ -495,8 +508,29 @@ describe("getNpmMinimumPackageAgeExclusions", () => {
},
});
const exclusions = getNpmMinimumPackageAgeExclusions();
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["react", "lodash"]);
});
it("should fall back to the legacy npm environment variable", () => {
process.env[legacyEnvVarName] = "lodash,react";
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["lodash", "react"]);
});
it("should read exclusions from the python config when the current ecosystem is py", () => {
setEcoSystem(ECOSYSTEM_PY);
configFileContent = JSON.stringify({
pip: {
minimumPackageAgeExclusions: ["requests", "urllib3"],
},
});
const exclusions = getMinimumPackageAgeExclusions();
assert.deepStrictEqual(exclusions, ["requests", "urllib3"]);
});
});

View file

@ -4,7 +4,7 @@ import {
getEcoSystem,
} from "../../config/settings.js";
import { npmInterceptorForUrl } from "./npm/npmInterceptor.js";
import { pipInterceptorForUrl } from "./pipInterceptor.js";
import { pipInterceptorForUrl } from "./pip/pipInterceptor.js";
/**
* @param {string} url

View file

@ -0,0 +1,33 @@
import { getMinimumPackageAgeExclusions, getEcoSystem } from "../../config/settings.js";
import { getEquivalentPackageNames } from "../../scanning/packageNameVariants.js";
/**
* Checks if a package name matches an exclusion pattern.
* Supports trailing wildcard (*) for prefix matching.
* @param {string} packageName
* @param {string} pattern
* @returns {boolean}
*/
export function matchesExclusionPattern(packageName, pattern) {
if (pattern.endsWith("/*")) {
return packageName.startsWith(pattern.slice(0, -1));
}
return packageName === pattern;
}
/**
* @param {string | undefined} packageName
* @returns {boolean}
*/
export function isExcludedFromMinimumPackageAge(packageName) {
if (!packageName) {
return false;
}
const exclusions = getMinimumPackageAgeExclusions();
const candidateNames = getEquivalentPackageNames(packageName, getEcoSystem());
return exclusions.some((pattern) =>
candidateNames.some((name) => matchesExclusionPattern(name, pattern))
);
}

View file

@ -196,17 +196,3 @@ export function getPackageNameFromMetadataResponse(body, headers) {
return undefined;
}
}
/**
* Checks if a package name matches an exclusion pattern.
* Supports trailing wildcard (*) for prefix matching.
* @param {string} packageName
* @param {string} pattern
* @returns {boolean}
*/
export function matchesExclusionPattern(packageName, pattern) {
if (pattern.endsWith("/*")) {
return packageName.startsWith(pattern.slice(0, -1));
}
return packageName === pattern;
}

View file

@ -1,6 +1,5 @@
import {
getNpmCustomRegistries,
getNpmMinimumPackageAgeExclusions,
skipMinimumPackageAge,
} from "../../../config/settings.js";
import { isMalwarePackage } from "../../../scanning/audit/index.js";
@ -8,12 +7,14 @@ import { interceptRequests } from "../interceptorBuilder.js";
import {
getPackageNameFromMetadataResponse,
isPackageInfoUrl,
matchesExclusionPattern,
modifyNpmInfoRequestHeaders,
modifyNpmInfoResponse,
} from "./modifyNpmInfo.js";
import { parseNpmPackageUrl } from "./parseNpmPackageUrl.js";
import { openNewPackagesDatabase } from "../../../scanning/newPackagesListCache.js";
import {
isExcludedFromMinimumPackageAge,
} from "../minimumPackageAgeExclusions.js";
const knownJsRegistries = [
"registry.npmjs.org",
@ -81,17 +82,6 @@ function buildNpmInterceptor(registry) {
});
}
/**
* @param {string} packageName
* @returns {boolean}
*/
function isExcludedFromMinimumPackageAge(packageName) {
const exclusions = getNpmMinimumPackageAgeExclusions();
return exclusions.some((pattern) =>
matchesExclusionPattern(packageName, pattern)
);
}
/**
* @param {Buffer} body
* @param {NodeJS.Dict<string | string[]> | undefined} headers

View file

@ -14,7 +14,7 @@ describe("npmInterceptor minimum package age", async () => {
getMinimumPackageAgeHours: () => minimumPackageAgeSettings,
skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
getNpmCustomRegistries: () => [],
getNpmMinimumPackageAgeExclusions: () => minimumPackageAgeExclusionsSetting,
getMinimumPackageAgeExclusions: () => minimumPackageAgeExclusionsSetting,
getEcoSystem: () => "js",
},
});

View file

@ -28,7 +28,7 @@ mock.module("../../../config/settings.js", {
setEcoSystem: () => {},
getMinimumPackageAgeHours: () => 24,
getNpmCustomRegistries: () => customRegistries,
getNpmMinimumPackageAgeExclusions: () => [],
getMinimumPackageAgeExclusions: () => [],
skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
},
});

View file

@ -0,0 +1,64 @@
/**
* @param {string} url
* @param {string} registry
* @returns {{packageName: string | undefined, version: string | undefined}}
*/
export function parsePipPackageFromUrl(url, registry) {
let packageName, version;
if (!registry || typeof url !== "string") {
return { packageName, version };
}
let urlObj;
try {
urlObj = new URL(url);
} catch {
return { packageName, version };
}
const lastSegment = urlObj.pathname.split("/").filter(Boolean).pop();
if (!lastSegment) {
return { packageName, version };
}
const filename = decodeURIComponent(lastSegment);
const wheelExtRe = /\.whl(?:\.metadata)?$/;
if (wheelExtRe.test(filename)) {
const base = filename.replace(wheelExtRe, "");
const firstDash = base.indexOf("-");
if (firstDash > 0) {
const dist = base.slice(0, firstDash);
const rest = base.slice(firstDash + 1);
const secondDash = rest.indexOf("-");
const rawVersion = secondDash >= 0 ? rest.slice(0, secondDash) : rest;
packageName = dist;
version = rawVersion;
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
}
const sdistExtWithMetadataRe = /\.(tar\.gz|zip|tar\.bz2|tar\.xz)(\.metadata)?$/i;
if (sdistExtWithMetadataRe.test(filename)) {
const base = filename.replace(sdistExtWithMetadataRe, "");
const lastDash = base.lastIndexOf("-");
if (lastDash > 0 && lastDash < base.length - 1) {
packageName = base.slice(0, lastDash);
version = base.slice(lastDash + 1);
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
}
return { packageName: undefined, version: undefined };
}

View file

@ -6,13 +6,25 @@ describe("pipInterceptor custom registries", async () => {
let malwareResponse = false;
let customRegistries = [];
mock.module("../../config/settings.js", {
mock.module("../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getMinimumPackageAgeExclusions: () => [],
getPipCustomRegistries: () => customRegistries,
skipMinimumPackageAge: () => false,
},
});
mock.module("../../scanning/audit/index.js", {
mock.module("../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: () => false,
}),
},
});
mock.module("../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async (packageName, version) => {
lastPackage = { packageName, version };
@ -30,10 +42,7 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.ok(
interceptor,
"Interceptor should be created for custom registry"
);
assert.ok(interceptor);
});
it("should parse package from custom registry URL", async () => {
@ -42,7 +51,7 @@ describe("pipInterceptor custom registries", async () => {
"https://my-custom-registry.example.com/packages/xx/yy/foobar-1.2.3.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
@ -58,7 +67,7 @@ describe("pipInterceptor custom registries", async () => {
"https://private-pypi.internal.com/packages/foo_bar-2.0.0-py3-none-any.whl";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
@ -82,11 +91,8 @@ describe("pipInterceptor custom registries", async () => {
const interceptor1 = pipInterceptorForUrl(url1);
const interceptor2 = pipInterceptorForUrl(url2);
assert.ok(interceptor1, "Interceptor should be created for first registry");
assert.ok(
interceptor2,
"Interceptor should be created for second registry"
);
assert.ok(interceptor1);
assert.ok(interceptor2);
});
it("should block malicious package from custom registry", async () => {
@ -97,21 +103,13 @@ describe("pipInterceptor custom registries", async () => {
"https://my-custom-registry.example.com/packages/malicious_package-1.0.0.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse, "Should contain a blockResponse");
assert.equal(
result.blockResponse.statusCode,
403,
"Block response should have status code 403"
);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain",
"Block response should have correct status message"
);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(result.blockResponse.message, "Forbidden - blocked by safe-chain");
malwareResponse = false;
});
@ -124,10 +122,7 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.ok(
interceptor,
"Interceptor should be created for known registry even with custom registries set"
);
assert.ok(interceptor);
await interceptor.handleRequest(url);
@ -143,11 +138,7 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.equal(
interceptor,
undefined,
"Interceptor should be undefined for unknown registry"
);
assert.equal(interceptor, undefined);
});
it("should handle empty custom registries array", () => {
@ -157,11 +148,7 @@ describe("pipInterceptor custom registries", async () => {
const interceptor = pipInterceptorForUrl(url);
assert.equal(
interceptor,
undefined,
"Interceptor should be undefined when no custom registries are configured"
);
assert.equal(interceptor, undefined);
});
it("should parse .whl.metadata from custom registry", async () => {
@ -170,7 +157,7 @@ describe("pipInterceptor custom registries", async () => {
"https://private-pypi.internal.com/packages/foo_bar-2.0.0-py3-none-any.whl.metadata";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
@ -186,7 +173,7 @@ describe("pipInterceptor custom registries", async () => {
"https://private-pypi.internal.com/packages/foo_bar-2.0.0.tar.gz.metadata";
const interceptor = pipInterceptorForUrl(url);
assert.ok(interceptor, "Interceptor should be created");
assert.ok(interceptor);
await interceptor.handleRequest(url);
@ -196,4 +183,3 @@ describe("pipInterceptor custom registries", async () => {
});
});
});

View file

@ -0,0 +1,80 @@
import {
getPipCustomRegistries,
skipMinimumPackageAge,
} from "../../../config/settings.js";
import { isMalwarePackage } from "../../../scanning/audit/index.js";
import { openNewPackagesDatabase } from "../../../scanning/newPackagesListCache.js";
import { interceptRequests } from "../interceptorBuilder.js";
import { isExcludedFromMinimumPackageAge } from "../minimumPackageAgeExclusions.js";
import { parsePipPackageFromUrl } from "./parsePipPackageUrl.js";
const knownPipRegistries = [
"files.pythonhosted.org",
"pypi.org",
"pypi.python.org",
"pythonhosted.org",
];
/**
* @param {string} url
* @returns {import("../interceptorBuilder.js").Interceptor | undefined}
*/
export function pipInterceptorForUrl(url) {
const customRegistries = getPipCustomRegistries();
const registries = [...knownPipRegistries, ...customRegistries];
const registry = registries.find((reg) => url.includes(reg));
if (registry) {
return buildPipInterceptor(registry);
}
return undefined;
}
/**
* @param {string} registry
* @returns {import("../interceptorBuilder.js").Interceptor | undefined}
*/
function buildPipInterceptor(registry) {
return interceptRequests(async (reqContext) => {
const { packageName, version } = parsePipPackageFromUrl(
reqContext.targetUrl,
registry
);
// PyPI treats hyphens and underscores as equivalent distribution names.
const hyphenName = packageName?.includes("_")
? packageName.replace(/_/g, "-")
: packageName;
const isMalicious =
await isMalwarePackage(packageName, version) ||
await isMalwarePackage(hyphenName, version);
if (isMalicious) {
reqContext.blockMalware(packageName, version);
return;
}
if (
packageName &&
version &&
!skipMinimumPackageAge() &&
!isExcludedFromMinimumPackageAge(packageName)
) {
const newPackagesDatabase = await openNewPackagesDatabase();
const isNewlyReleased = newPackagesDatabase.isNewlyReleasedPackage(
packageName,
version
);
if (isNewlyReleased) {
reqContext.blockMinimumAgeRequest(
packageName,
version,
`Forbidden - blocked by safe-chain direct download minimum package age (${packageName}@${version})`
);
}
}
});
}

View file

@ -0,0 +1,103 @@
import { describe, it, mock } from "node:test";
import assert from "node:assert";
describe("pipInterceptor minimum package age", async () => {
let skipMinimumPackageAgeSetting = false;
let newlyReleasedPackageResponse = false;
let minimumPackageAgeExclusionsSetting = [];
mock.module("../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async () => false,
},
});
mock.module("../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: (packageName, version) => {
return newlyReleasedPackageResponse &&
(packageName === "foo-bar" ||
packageName === "foo_bar" ||
packageName === "foo.bar") &&
version === "2.0.0";
},
}),
},
});
mock.module("../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getMinimumPackageAgeExclusions: () => minimumPackageAgeExclusionsSetting,
getPipCustomRegistries: () => [],
skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
},
});
const { pipInterceptorForUrl } = await import("./pipInterceptor.js");
it("should block newly released package downloads", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl";
newlyReleasedPackageResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain direct download minimum package age (foo_bar@2.0.0)"
);
newlyReleasedPackageResponse = false;
});
it("should not block newly released package downloads when skipMinimumPackageAge is enabled", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl";
newlyReleasedPackageResponse = true;
skipMinimumPackageAgeSetting = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
skipMinimumPackageAgeSetting = false;
newlyReleasedPackageResponse = false;
});
it("should not block newly released package downloads when the package is excluded", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl";
newlyReleasedPackageResponse = true;
minimumPackageAgeExclusionsSetting = ["foo-bar"];
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
minimumPackageAgeExclusionsSetting = [];
newlyReleasedPackageResponse = false;
});
it("should not block newly released package downloads when a dot-name package matches a hyphen exclusion", async () => {
const url =
"https://files.pythonhosted.org/packages/xx/yy/foo.bar-2.0.0.tar.gz";
newlyReleasedPackageResponse = true;
minimumPackageAgeExclusionsSetting = ["foo-bar"];
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.equal(result.blockResponse, undefined);
minimumPackageAgeExclusionsSetting = [];
newlyReleasedPackageResponse = false;
});
});

View file

@ -5,7 +5,7 @@ describe("pipInterceptor", async () => {
let lastPackage;
let malwareResponse = false;
mock.module("../../scanning/audit/index.js", {
mock.module("../../../scanning/audit/index.js", {
namedExports: {
isMalwarePackage: async (packageName, version) => {
lastPackage = { packageName, version };
@ -14,10 +14,27 @@ describe("pipInterceptor", async () => {
},
});
mock.module("../../../scanning/newPackagesListCache.js", {
namedExports: {
openNewPackagesDatabase: async () => ({
isNewlyReleasedPackage: () => false,
}),
},
});
mock.module("../../../config/settings.js", {
namedExports: {
ECOSYSTEM_PY: "py",
getEcoSystem: () => "py",
getMinimumPackageAgeExclusions: () => [],
getPipCustomRegistries: () => [],
skipMinimumPackageAge: () => false,
},
});
const { pipInterceptorForUrl } = await import("./pipInterceptor.js");
const parserCases = [
// Valid pip URLs
{
url: "https://files.pythonhosted.org/packages/xx/yy/foobar-1.2.3.tar.gz",
expected: { packageName: "foobar", version: "1.2.3" },
@ -35,7 +52,6 @@ describe("pipInterceptor", async () => {
expected: { packageName: "foo-bar", version: "2.0.0" },
},
{
// Poetry preflight metadata alongside wheel (.whl.metadata)
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0-py3-none-any.whl.metadata",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
@ -52,7 +68,6 @@ describe("pipInterceptor", async () => {
expected: { packageName: "foo-bar", version: "2.0.0b1" },
},
{
// sdist with metadata sidecar (.tar.gz.metadata)
url: "https://files.pythonhosted.org/packages/xx/yy/foo_bar-2.0.0.tar.gz.metadata",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
@ -76,7 +91,6 @@ describe("pipInterceptor", async () => {
url: "https://pypi.org/packages/source/f/foo_bar/foo_bar-2.0.0-cp38-cp38-manylinux1_x86_64.whl",
expected: { packageName: "foo-bar", version: "2.0.0" },
},
// Invalid pip URLs
{
url: "https://pypi.org/simple/",
expected: { packageName: undefined, version: undefined },
@ -98,10 +112,7 @@ describe("pipInterceptor", async () => {
parserCases.forEach(({ url, expected }, index) => {
it(`should parse URL ${index + 1}: ${url}`, async () => {
const interceptor = pipInterceptorForUrl(url);
assert.ok(
interceptor,
"Interceptor should be created for known npm registry"
);
assert.ok(interceptor, "Interceptor should be created for known pip registry");
await interceptor.handleRequest(url);
@ -111,14 +122,8 @@ describe("pipInterceptor", async () => {
it("should not create interceptor for unknown registry", () => {
const url = "https://example.com/packages/xx/yy/foobar-1.2.3.tar.gz";
const interceptor = pipInterceptorForUrl(url);
assert.equal(
interceptor,
undefined,
"Interceptor should be undefined for unknown registry"
);
assert.equal(interceptor, undefined);
});
it("should block malicious package", async () => {
@ -127,19 +132,15 @@ describe("pipInterceptor", async () => {
malwareResponse = true;
const interceptor = pipInterceptorForUrl(url);
const result = await interceptor.handleRequest(url);
assert.ok(result.blockResponse, "Should contain a blockResponse");
assert.equal(
result.blockResponse.statusCode,
403,
"Block response should have status code 403"
);
assert.ok(result.blockResponse);
assert.equal(result.blockResponse.statusCode, 403);
assert.equal(
result.blockResponse.message,
"Forbidden - blocked by safe-chain",
"Block response should have correct status message"
"Forbidden - blocked by safe-chain"
);
malwareResponse = false;
});
});

View file

@ -1,132 +0,0 @@
import { getPipCustomRegistries } from "../../config/settings.js";
import { isMalwarePackage } from "../../scanning/audit/index.js";
import { interceptRequests } from "./interceptorBuilder.js";
const knownPipRegistries = [
"files.pythonhosted.org",
"pypi.org",
"pypi.python.org",
"pythonhosted.org",
];
/**
* @param {string} url
* @returns {import("./interceptorBuilder.js").Interceptor | undefined}
*/
export function pipInterceptorForUrl(url) {
const customRegistries = getPipCustomRegistries();
const registries = [...knownPipRegistries, ...customRegistries];
const registry = registries.find((reg) => url.includes(reg));
if (registry) {
return buildPipInterceptor(registry);
}
return undefined;
}
/**
* @param {string} registry
* @returns {import("./interceptorBuilder.js").Interceptor | undefined}
*/
function buildPipInterceptor(registry) {
return interceptRequests(async (reqContext) => {
const { packageName, version } = parsePipPackageFromUrl(
reqContext.targetUrl,
registry
);
// Normalize underscores to hyphens for DB matching, as PyPI allows underscores in distribution names.
// Per python, packages that differ only by hyphen vs underscore are considered the same.
const hyphenName = packageName?.includes("_") ? packageName.replace(/_/g, "-") : packageName;
const isMalicious =
await isMalwarePackage(packageName, version)
|| await isMalwarePackage(hyphenName, version);
if (isMalicious) {
reqContext.blockMalware(packageName, version);
}
});
}
/**
* @param {string} url
* @param {string} registry
* @returns {{packageName: string | undefined, version: string | undefined}}
*/
function parsePipPackageFromUrl(url, registry) {
let packageName, version;
// Basic validation
if (!registry || typeof url !== "string") {
return { packageName, version };
}
// Quick sanity check on the URL + parse
let urlObj;
try {
urlObj = new URL(url);
} catch {
return { packageName, version };
}
// Get the last path segment (filename) and decode it (strip query & fragment automatically)
const lastSegment = urlObj.pathname.split("/").filter(Boolean).pop();
if (!lastSegment) {
return { packageName, version };
}
const filename = decodeURIComponent(lastSegment);
// Parse Python package downloads from PyPI/pythonhosted.org
// Example wheel: https://files.pythonhosted.org/packages/xx/yy/requests-2.28.1-py3-none-any.whl
// Example sdist: https://files.pythonhosted.org/packages/xx/yy/requests-2.28.1.tar.gz
// Wheel (.whl) and Poetry's preflight metadata (.whl.metadata)
// Examples:
// foo_bar-2.0.0-py3-none-any.whl
// foo_bar-2.0.0-py3-none-any.whl.metadata
const wheelExtRe = /\.whl(?:\.metadata)?$/;
const wheelExtMatch = filename.match(wheelExtRe);
if (wheelExtMatch) {
const base = filename.replace(wheelExtRe, "");
const firstDash = base.indexOf("-");
if (firstDash > 0) {
const dist = base.slice(0, firstDash); // may contain underscores
const rest = base.slice(firstDash + 1); // version + the rest of tags
const secondDash = rest.indexOf("-");
const rawVersion = secondDash >= 0 ? rest.slice(0, secondDash) : rest;
packageName = dist;
version = rawVersion;
// Reject "latest" as it's a placeholder, not a real version
// When version is "latest", this signals the URL doesn't contain actual version info
// Returning undefined allows the request (see registryProxy.js isAllowedUrl)
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
}
// Source dist (sdist) and potential metadata sidecars (e.g., .tar.gz.metadata)
const sdistExtWithMetadataRe = /\.(tar\.gz|zip|tar\.bz2|tar\.xz)(\.metadata)?$/i;
const sdistExtMatch = filename.match(sdistExtWithMetadataRe);
if (sdistExtMatch) {
const base = filename.replace(sdistExtWithMetadataRe, "");
const lastDash = base.lastIndexOf("-");
if (lastDash > 0 && lastDash < base.length - 1) {
packageName = base.slice(0, lastDash);
version = base.slice(lastDash + 1);
// Reject "latest" as it's a placeholder, not a real version
// When version is "latest", this signals the URL doesn't contain actual version info
// Returning undefined allows the request (see registryProxy.js isAllowedUrl)
if (version === "latest" || !packageName || !version) {
return { packageName: undefined, version: undefined };
}
return { packageName, version };
}
}
// Unknown file type or invalid
return { packageName: undefined, version: undefined };
}

View file

@ -174,10 +174,18 @@ describe("newPackagesDatabase", async () => {
assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
});
it("returns false for all packages when ecosystem is not JS", async () => {
it("supports package checks for the python ecosystem", async () => {
ecosystem = "py";
fetchedList = [
{
source: "pypi",
package_name: "foo",
version: "1.0.0",
released_on: hoursAgo(1),
},
];
const db = await openNewPackagesDatabase();
assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), false);
assert.strictEqual(db.isNewlyReleasedPackage("foo", "1.0.0"), true);
});
});

View file

@ -4,10 +4,11 @@ import {
ECOSYSTEM_JS,
ECOSYSTEM_PY,
} from "../config/settings.js";
import { getEquivalentPackageNames } from "./packageNameVariants.js";
/**
* @typedef {Object} NewPackagesDatabase
* @property {function(string, string): boolean} isNewlyReleasedPackage
* @property {function(string | undefined, string | undefined): boolean} isNewlyReleasedPackage
*/
/**
@ -33,21 +34,28 @@ function getCurrentFeedSource() {
* @returns {NewPackagesDatabase}
*/
export function buildNewPackagesDatabase(newPackagesList) {
const ecosystem = getEcoSystem();
/**
* @param {string} name
* @param {string} version
* @param {string | undefined} name
* @param {string | undefined} version
* @returns {boolean}
*/
function isNewlyReleasedPackage(name, version) {
if (!name || !version) {
return false;
}
const cutOff = new Date(
new Date().getTime() - getMinimumPackageAgeHours() * 3600 * 1000
);
const expectedSource = getCurrentFeedSource();
const candidateNames = getEquivalentPackageNames(name, ecosystem);
const entry = newPackagesList.find(
(pkg) =>
(!pkg.source || pkg.source.toLowerCase() === expectedSource) &&
pkg.package_name === name &&
candidateNames.includes(pkg.package_name) &&
pkg.version === version
);

View file

@ -50,6 +50,15 @@ describe("buildNewPackagesDatabase", () => {
assert.strictEqual(db.isNewlyReleasedPackage("not-there", "1.0.0"), false);
});
it("returns false when name or version is undefined", () => {
const db = buildNewPackagesDatabase([
{ package_name: "foo", version: "1.0.0", released_on: hoursAgo(1) },
]);
assert.strictEqual(db.isNewlyReleasedPackage(undefined, "1.0.0"), false);
assert.strictEqual(db.isNewlyReleasedPackage("foo", undefined), false);
});
it("returns false for a known package but different version", () => {
const db = buildNewPackagesDatabase([
{ package_name: "foo", version: "2.0.0", released_on: hoursAgo(1) },
@ -96,5 +105,54 @@ describe("buildNewPackagesDatabase", () => {
minimumPackageAgeHours = 24; // reset
});
it("matches underscore request names against hyphen feed names for python", () => {
ecosystem = "py";
const db = buildNewPackagesDatabase([
{ source: "pypi", package_name: "foo-bar", version: "1.0.0", released_on: hoursAgo(1) },
]);
assert.strictEqual(db.isNewlyReleasedPackage("foo_bar", "1.0.0"), true);
ecosystem = "js";
});
it("matches hyphen request names against underscore feed names for python", () => {
ecosystem = "py";
const db = buildNewPackagesDatabase([
{ source: "pypi", package_name: "foo_bar", version: "1.0.0", released_on: hoursAgo(1) },
]);
assert.strictEqual(db.isNewlyReleasedPackage("foo-bar", "1.0.0"), true);
ecosystem = "js";
});
it("matches dot request names against hyphen feed names for python", () => {
ecosystem = "py";
const db = buildNewPackagesDatabase([
{ source: "pypi", package_name: "foo-bar", version: "1.0.0", released_on: hoursAgo(1) },
]);
assert.strictEqual(db.isNewlyReleasedPackage("foo.bar", "1.0.0"), true);
ecosystem = "js";
});
it("matches underscore request names against dot feed names for python", () => {
ecosystem = "py";
const db = buildNewPackagesDatabase([
{ source: "pypi", package_name: "foo.bar", version: "1.0.0", released_on: hoursAgo(1) },
]);
assert.strictEqual(db.isNewlyReleasedPackage("foo_bar", "1.0.0"), true);
ecosystem = "js";
});
});
});

View file

@ -8,7 +8,6 @@ import {
getNewPackagesListVersionPath,
} from "../config/configFile.js";
import { ui } from "../environment/userInteraction.js";
import { getEcoSystem, ECOSYSTEM_JS } from "../config/settings.js";
import { buildNewPackagesDatabase } from "./newPackagesDatabaseBuilder.js";
import { warnOnceAboutUnavailableDatabase } from "./newPackagesDatabaseWarnings.js";
@ -28,11 +27,6 @@ export async function openNewPackagesDatabase() {
return cachedNewPackagesDatabase;
}
if (getEcoSystem() !== ECOSYSTEM_JS) {
cachedNewPackagesDatabase = { isNewlyReleasedPackage: () => false };
return cachedNewPackagesDatabase;
}
/** @type {import("../api/aikido.js").NewPackageEntry[]} */
let newPackagesList;

View file

@ -0,0 +1,18 @@
import { ECOSYSTEM_PY } from "../config/settings.js";
/**
* @param {string} packageName
* @param {string} ecosystem
* @returns {string[]}
*/
export function getEquivalentPackageNames(packageName, ecosystem) {
if (ecosystem !== ECOSYSTEM_PY) {
return [packageName];
}
const hyphenName = packageName.replaceAll(/[_.-]/g, "-");
const underscoreName = packageName.replaceAll(/[._-]/g, "_");
const dotName = packageName.replaceAll(/[_.-]/g, ".");
return [...new Set([packageName, hyphenName, underscoreName, dotName])];
}