Skip to content

Commit e23080b

Browse files
fengmk2fengmk2
committed
feat: add unpkg-white-list to detect sync unpkg files or not
see https://github.com/cnpm/unpkg-white-list
1 parent c5c6145 commit e23080b

File tree

8 files changed

+297
-9
lines changed

8 files changed

+297
-9
lines changed

‎app.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import path from 'path';
22
import { readFile } from 'fs/promises';
33
import { Application } from 'egg';
44
import { ChangesStreamService } from './app/core/service/ChangesStreamService';
5+
56
declare module 'egg' {
67
interface Application {
78
binaryHTML: string;

‎app/core/event/SyncPackageVersionFile.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { Event, Inject } from '@eggjs/tegg';
22
import {
33
EggAppConfig,
44
} from 'egg';
5+
import { ForbiddenError } from 'egg-errors';
56
import { PACKAGE_VERSION_ADDED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED } from './index';
67
import { getScopeAndName } from '../../common/PackageUtil';
78
import { PackageManagerService } from '../service/PackageManagerService';
@@ -25,7 +26,15 @@ class SyncPackageVersionFileEvent {
2526
const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
2627
scope, name, version);
2728
if (!packageVersion) return;
28-
await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
29+
try {
30+
await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
31+
} catch (err) {
32+
if (err instanceof ForbiddenError) {
33+
// ignore it
34+
return;
35+
}
36+
throw err;
37+
}
2938
}
3039

3140
protected async syncPackageReadmeToLatestVersion(fullname: string) {

‎app/core/service/PackageVersionFileService.ts‎

Lines changed: 65 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,34 @@ import {
77
SingletonProto,
88
Inject,
99
} from '@eggjs/tegg';
10+
import { ConflictError, ForbiddenError } from 'egg-errors';
11+
import semver from 'semver';
1012
import { AbstractService } from '../../common/AbstractService';
1113
import {
1214
calculateIntegrity,
15+
getFullname,
1316
} from '../../common/PackageUtil';
1417
import { createTempDir, mimeLookup } from '../../common/FileUtil';
1518
import {
1619
PackageRepository,
1720
} from '../../repository/PackageRepository';
1821
import { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository';
22+
import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
1923
import { DistRepository } from '../../repository/DistRepository';
2024
import { PackageVersionFile } from '../entity/PackageVersionFile';
2125
import { PackageVersion } from '../entity/PackageVersion';
2226
import { Package } from '../entity/Package';
2327
import { PackageManagerService } from './PackageManagerService';
2428
import { CacheAdapter } from '../../common/adapter/CacheAdapter';
25-
import { ConflictError } from 'egg-errors';
29+
30+
const unpkgWhiteListUrl = 'https://github.com/cnpm/unpkg-white-list';
2631

2732
@SingletonProto({
2833
accessLevel: AccessLevel.PUBLIC,
2934
})
3035
export class PackageVersionFileService extends AbstractService {
36+
@Inject()
37+
private readonly packageVersionRepository: PackageVersionRepository;
3138
@Inject()
3239
private readonly packageRepository: PackageRepository;
3340
@Inject()
@@ -39,6 +46,12 @@ export class PackageVersionFileService extends AbstractService {
3946
@Inject()
4047
private readonly cacheAdapter: CacheAdapter;
4148

49+
#unpkgWhiteListCurrentVersion: string = '';
50+
#unpkgWhiteListAllowPackages: Record<string, {
51+
version: string;
52+
}> = {};
53+
#unpkgWhiteListAllowScopes: string[] = [];
54+
4255
async listPackageVersionFiles(pkgVersion: PackageVersion, directory: string) {
4356
await this.#ensurePackageVersionFilesSync(pkgVersion);
4457
return await this.packageVersionFileRepository.listPackageVersionFiles(pkgVersion.packageVersionId, directory);
@@ -54,16 +67,57 @@ export class PackageVersionFileService extends AbstractService {
5467
async #ensurePackageVersionFilesSync(pkgVersion: PackageVersion) {
5568
const hasFiles = await this.packageVersionFileRepository.hasPackageVersionFiles(pkgVersion.packageVersionId);
5669
if (!hasFiles) {
57-
const lockRes = await this.cacheAdapter.usingLock(`${pkgVersion.packageVersionId}:syncFiles`, 60, async () => {
70+
const lockName = `${pkgVersion.packageVersionId}:syncFiles`;
71+
const lockRes = await this.cacheAdapter.usingLock(lockName, 60, async () => {
5872
await this.syncPackageVersionFiles(pkgVersion);
5973
});
6074
// lock fail
6175
if (!lockRes) {
62-
this.logger.warn('[package:version:syncPackageVersionFiles] check lock fail');
76+
this.logger.warn('[package:version:syncPackageVersionFiles] check lock:%s fail', lockName);
6377
throw new ConflictError('Package version file sync is currently in progress. Please try again later.');
6478
}
6579
}
80+
}
81+
82+
async updateUnpkgWhiteList() {
83+
if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
84+
const whiteListScope = '';
85+
const whiteListPackageName = 'unpkg-white-list';
86+
const whiteListPackageVersion = await this.packageVersionRepository.findVersionByTag(
87+
whiteListScope, whiteListPackageName, 'latest');
88+
if (!whiteListPackageVersion) return;
89+
if (this.#unpkgWhiteListCurrentVersion === whiteListPackageVersion) return;
90+
91+
// update the new version white list
92+
const { manifest } = await this.packageManagerService.showPackageVersionManifest(
93+
whiteListScope, whiteListPackageName, whiteListPackageVersion, false, true);
94+
if (!manifest) return;
95+
this.#unpkgWhiteListCurrentVersion = manifest.version;
96+
this.#unpkgWhiteListAllowPackages = manifest.allowPackages ?? {} as any;
97+
this.#unpkgWhiteListAllowScopes = manifest.allowScopes ?? [] as any;
98+
this.logger.info('[PackageVersionFileService.updateUnpkgWhiteList] version:%s, total %s packages, %s scopes',
99+
whiteListPackageVersion,
100+
Object.keys(this.#unpkgWhiteListAllowPackages).length,
101+
this.#unpkgWhiteListAllowScopes.length,
102+
);
103+
}
66104

105+
async #checkPackageVersionInUnpkgWhiteList(pkgScope: string, pkgName: string, pkgVersion: string) {
106+
if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
107+
await this.updateUnpkgWhiteList();
108+
109+
// check allow scopes
110+
if (this.#unpkgWhiteListAllowScopes.includes(pkgScope)) return;
111+
112+
// check allow packages
113+
const fullname = getFullname(pkgScope, pkgName);
114+
const pkgConfig = this.#unpkgWhiteListAllowPackages[fullname];
115+
if (!pkgConfig) {
116+
throw new ForbiddenError(`"${fullname}" is not allow to unpkg files, see ${unpkgWhiteListUrl}`);
117+
}
118+
if (!pkgConfig.version || !semver.satisfies(pkgVersion, pkgConfig.version)) {
119+
throw new ForbiddenError(`"${fullname}@${pkgVersion}" not satisfies "${pkgConfig.version}" to unpkg files, see ${unpkgWhiteListUrl}`);
120+
}
67121
}
68122

69123
// 基于 latest version 同步 package readme
@@ -113,8 +167,16 @@ export class PackageVersionFileService extends AbstractService {
113167

114168
async syncPackageVersionFiles(pkgVersion: PackageVersion) {
115169
const files: PackageVersionFile[] = [];
170+
// must set enableUnpkg and enableSyncUnpkgFiles = true both
171+
if (!this.config.cnpmcore.enableUnpkg) return files;
172+
if (!this.config.cnpmcore.enableSyncUnpkgFiles) return files;
173+
116174
const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
117175
if (!pkg) return files;
176+
177+
// check unpkg white list
178+
await this.#checkPackageVersionInUnpkgWhiteList(pkg.scope, pkg.name, pkgVersion.version);
179+
118180
const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${pkgVersion.version}_${randomUUID()}`;
119181
const tmpdir = await createTempDir(this.config.dataDir, dirname);
120182
const tarFile = `${tmpdir}.tgz`;

‎app/port/config.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,10 @@ export type CnpmcoreConfig = {
149149
* enable sync unpkg files
150150
*/
151151
enableSyncUnpkgFiles: boolean;
152+
/**
153+
* enable sync unpkg files from the white list, https://github.com/cnpm/unpkg-white-list
154+
*/
155+
enableSyncUnpkgFilesWhiteList: boolean;
152156
/**
153157
* enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag.
154158
* in most cases, you should set to false to keep the same behavior as source registry.

‎app/port/controller/PackageVersionFileController.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,12 +149,9 @@ export class PackageVersionFileController extends AbstractController {
149149

150150
if (!file) {
151151
const possibleFile = await this.#searchPossibleEntries(packageVersion, path);
152-
153152
if (possibleFile) {
154153
const route = `/${fullname}/${versionSpec}/files${possibleFile.path}${hasMeta ? '?meta' : ''}`;
155-
156154
ctx.redirect(route);
157-
158155
return;
159156
}
160157

‎app/port/controller/package/SavePackageVersionController.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ export class SavePackageVersionController extends AbstractController {
221221
const registry = await this.registryManagerService.ensureSelfRegistry();
222222

223223
let packageVersionEntity: PackageVersionEntity | undefined;
224+
const lockName = `${pkg.name}:publish`;
224225
const lockRes = await this.cacheAdapter.usingLock(`${pkg.name}:publish`, 60, async () => {
225226
packageVersionEntity = await this.packageManagerService.publish({
226227
scope,
@@ -240,7 +241,7 @@ export class SavePackageVersionController extends AbstractController {
240241

241242
// lock fail
242243
if (!lockRes) {
243-
this.logger.warn('[package:version:add] check lock fail');
244+
this.logger.warn('[package:version:add] check lock:%s fail', lockName);
244245
throw new ConflictError('Unable to create the publication lock, please try again later.');
245246
}
246247

‎config/config.default.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
5454
redirectNotFound: true,
5555
enableUnpkg: true,
5656
enableSyncUnpkgFiles: true,
57+
enableSyncUnpkgFilesWhiteList: false,
5758
strictSyncSpecivicVersion: false,
5859
enableElasticsearch: !!process.env.CNPMCORE_CONFIG_ENABLE_ES,
5960
elasticsearchIndex: 'cnpmcore_packages',

0 commit comments

Comments
 (0)