From 47f5c3a625532a24748dfb92c6877ffe5705b96f Mon Sep 17 00:00:00 2001 From: Vladimir Date: Thu, 3 Aug 2023 12:21:32 +0200 Subject: [PATCH] refactor: split executor based on module resolution (#3878) --- packages/vitest/src/runtime/execute.ts | 3 + .../vitest/src/runtime/external-executor.ts | 576 ++---------------- .../src/runtime/vm/commonjs-executor.ts | 226 +++++++ .../vitest/src/runtime/vm/esm-executor.ts | 211 +++++++ packages/vitest/src/runtime/vm/file-map.ts | 24 + packages/vitest/src/runtime/vm/types.ts | 72 +++ packages/vitest/src/runtime/vm/utils.ts | 33 + 7 files changed, 612 insertions(+), 533 deletions(-) create mode 100644 packages/vitest/src/runtime/vm/commonjs-executor.ts create mode 100644 packages/vitest/src/runtime/vm/esm-executor.ts create mode 100644 packages/vitest/src/runtime/vm/file-map.ts create mode 100644 packages/vitest/src/runtime/vm/types.ts create mode 100644 packages/vitest/src/runtime/vm/utils.ts diff --git a/packages/vitest/src/runtime/execute.ts b/packages/vitest/src/runtime/execute.ts index 271d2e0a6275..d379a1b57165 100644 --- a/packages/vitest/src/runtime/execute.ts +++ b/packages/vitest/src/runtime/execute.ts @@ -11,6 +11,7 @@ import { distDir } from '../paths' import { getWorkerState } from '../utils/global' import { VitestMocker } from './mocker' import { ExternalModulesExecutor } from './external-executor' +import { FileMap } from './vm/file-map' const entryUrl = pathToFileURL(resolve(distDir, 'entry.js')).href @@ -39,6 +40,7 @@ let _viteNode: { export const packageCache = new Map() export const moduleCache = new ModuleCacheMap() export const mockMap: MockMap = new Map() +export const fileMap = new FileMap() export async function startViteNode(options: ContextExecutorOptions) { if (_viteNode) @@ -174,6 +176,7 @@ export class VitestExecutor extends ViteNodeRunner { else { this.externalModules = new ExternalModulesExecutor({ ...options, + fileMap, context: options.context, packageCache: options.packageCache, }) diff --git a/packages/vitest/src/runtime/external-executor.ts b/packages/vitest/src/runtime/external-executor.ts index 85e956104627..bf90480ad3b8 100644 --- a/packages/vitest/src/runtime/external-executor.ts +++ b/packages/vitest/src/runtime/external-executor.ts @@ -3,277 +3,62 @@ import vm from 'node:vm' import { fileURLToPath, pathToFileURL } from 'node:url' import { dirname } from 'node:path' -import { Module as _Module, createRequire } from 'node:module' -import { readFileSync, statSync } from 'node:fs' -import { basename, extname, join, normalize } from 'pathe' -import { getCachedData, isNodeBuiltin, isPrimitive, setCacheData } from 'vite-node/utils' -import { CSS_LANGS_RE, KNOWN_ASSET_RE } from 'vite-node/constants' -import { getColors } from '@vitest/utils' +import { statSync } from 'node:fs' +import { extname, join, normalize } from 'pathe' +import { getCachedData, isNodeBuiltin, setCacheData } from 'vite-node/utils' import type { ExecuteOptions } from './execute' - -// need to copy paste types for vm -// because they require latest @types/node which we don't bundle - -interface ModuleEvaluateOptions { - timeout?: vm.RunningScriptOptions['timeout'] | undefined - breakOnSigint?: vm.RunningScriptOptions['breakOnSigint'] | undefined -} - -type ModuleLinker = (specifier: string, referencingModule: VMModule, extra: { assert: Object }) => VMModule | Promise -type ModuleStatus = 'unlinked' | 'linking' | 'linked' | 'evaluating' | 'evaluated' | 'errored' -declare class VMModule { - dependencySpecifiers: readonly string[] - error: any - identifier: string - context: vm.Context - namespace: Object - status: ModuleStatus - evaluate(options?: ModuleEvaluateOptions): Promise - link(linker: ModuleLinker): Promise -} -interface SyntheticModuleOptions { - /** - * String used in stack traces. - * @default 'vm:module(i)' where i is a context-specific ascending index. - */ - identifier?: string | undefined - /** - * The contextified object as returned by the `vm.createContext()` method, to compile and evaluate this module in. - */ - context?: vm.Context | undefined -} -declare class VMSyntheticModule extends VMModule { - /** - * Creates a new `SyntheticModule` instance. - * @param exportNames Array of names that will be exported from the module. - * @param evaluateCallback Called when the module is evaluated. - */ - constructor(exportNames: string[], evaluateCallback: (this: VMSyntheticModule) => void, options?: SyntheticModuleOptions) - /** - * This method is used after the module is linked to set the values of exports. - * If it is called before the module is linked, an `ERR_VM_MODULE_STATUS` error will be thrown. - * @param name - * @param value - */ - setExport(name: string, value: any): void -} - -declare interface ImportModuleDynamically { - (specifier: string, script: VMModule, importAssertions: Object): VMModule | Promise -} - -interface SourceTextModuleOptions { - identifier?: string | undefined - cachedData?: vm.ScriptOptions['cachedData'] | undefined - context?: vm.Context | undefined - lineOffset?: vm.BaseOptions['lineOffset'] | undefined - columnOffset?: vm.BaseOptions['columnOffset'] | undefined - /** - * Called during evaluation of this module to initialize the `import.meta`. - */ - initializeImportMeta?: ((meta: ImportMeta, module: VMSourceTextModule) => void) | undefined - importModuleDynamically?: ImportModuleDynamically -} -declare class VMSourceTextModule extends VMModule { - /** - * Creates a new `SourceTextModule` instance. - * @param code JavaScript Module code to parse - */ - constructor(code: string, options?: SourceTextModuleOptions) -} +import type { VMModule, VMSyntheticModule } from './vm/types' +import { CommonjsExecutor } from './vm/commonjs-executor' +import type { FileMap } from './vm/file-map' +import { EsmExecutor } from './vm/esm-executor' +import { interopCommonJsModule } from './vm/utils' const SyntheticModule: typeof VMSyntheticModule = (vm as any).SyntheticModule -const SourceTextModule: typeof VMSourceTextModule = (vm as any).SourceTextModule - -interface PrivateNodeModule extends NodeModule { - _compile(code: string, filename: string): void -} - -const _require = createRequire(import.meta.url) const nativeResolve = import.meta.resolve! -const dataURIRegex - = /^data:(?text\/javascript|application\/json|application\/wasm)(?:;(?charset=utf-8|base64))?,(?.*)$/ - export interface ExternalModulesExecutorOptions extends ExecuteOptions { context: vm.Context + fileMap: FileMap packageCache: Map } // TODO: improve Node.js strict mode support in #2854 export class ExternalModulesExecutor { - private requireCache: Record = Object.create(null) - private builtinCache: Record = Object.create(null) - private moduleCache = new Map>() - private extensions: Record unknown> = Object.create(null) - - private esmLinkMap = new WeakMap>() + private cjs: CommonjsExecutor + private esm: EsmExecutor private context: vm.Context - - private fsCache = new Map() - private fsBufferCache = new Map() - - private Module: typeof _Module - private primitives: { - Object: typeof Object - Array: typeof Array - Error: typeof Error - } + private fs: FileMap constructor(private options: ExternalModulesExecutorOptions) { this.context = options.context - const primitives = vm.runInContext('({ Object, Array, Error })', this.context) as { - Object: typeof Object - Array: typeof Array - Error: typeof Error - } - this.primitives = primitives - - // eslint-disable-next-line @typescript-eslint/no-this-alias - const executor = this - - // primitive implementation, some fields are not filled yet, like "paths" - #2854 - this.Module = class Module { - exports: any - isPreloading = false - require: NodeRequire - id: string - filename: string - loaded: boolean - parent: null | Module | undefined - children: Module[] = [] - path: string - paths: string[] = [] - - constructor(id: string, parent?: Module) { - this.exports = primitives.Object.create(Object.prototype) - this.require = Module.createRequire(id) - // in our case the path should always be resolved already - this.path = dirname(id) - this.id = id - this.filename = id - this.loaded = false - this.parent = parent - } - - _compile(code: string, filename: string) { - const cjsModule = Module.wrap(code) - const script = new vm.Script(cjsModule, { - filename, - importModuleDynamically: executor.importModuleDynamically, - } as any) - // @ts-expect-error mark script with current identifier - script.identifier = filename - const fn = script.runInContext(executor.context) - const __dirname = dirname(filename) - executor.requireCache[filename] = this - try { - fn(this.exports, this.require, this, filename, __dirname) - return this.exports - } - finally { - this.loaded = true - } - } - - // exposed for external use, Node.js does the opposite - static _load = (request: string, parent: Module | undefined, _isMain: boolean) => { - const require = Module.createRequire(parent?.filename ?? request) - return require(request) - } - - static wrap = (script: string) => { - return Module.wrapper[0] + script + Module.wrapper[1] - } - - static wrapper = new primitives.Array( - '(function (exports, require, module, __filename, __dirname) { ', - '\n});', - ) - - static builtinModules = _Module.builtinModules - static findSourceMap = _Module.findSourceMap - static SourceMap = _Module.SourceMap - static syncBuiltinESMExports = _Module.syncBuiltinESMExports - - static _cache = executor.moduleCache - static _extensions = executor.extensions - - static createRequire = (filename: string) => { - return executor.createRequire(filename) - } - - static runMain = () => { - throw new primitives.Error('[vitest] "runMain" is not implemented.') - } - - // @ts-expect-error not typed - static _resolveFilename = _Module._resolveFilename - // @ts-expect-error not typed - static _findPath = _Module._findPath - // @ts-expect-error not typed - static _initPaths = _Module._initPaths - // @ts-expect-error not typed - static _preloadModules = _Module._preloadModules - // @ts-expect-error not typed - static _resolveLookupPaths = _Module._resolveLookupPaths - // @ts-expect-error not typed - static globalPaths = _Module.globalPaths - // eslint-disable-next-line @typescript-eslint/prefer-ts-expect-error - // @ts-ignore not typed in lower versions - static isBuiltin = _Module.isBuiltin - - static Module = Module - } - - this.extensions['.js'] = this.requireJs - this.extensions['.json'] = this.requireJson - } - - private requireJs = (m: NodeModule, filename: string) => { - const content = this.readFile(filename) - ;(m as PrivateNodeModule)._compile(content, filename) - } - - private requireJson = (m: NodeModule, filename: string) => { - const code = this.readFile(filename) - m.exports = JSON.parse(code) + this.fs = options.fileMap + this.esm = new EsmExecutor(this, { + context: this.context, + }) + this.cjs = new CommonjsExecutor({ + context: this.context, + importModuleDynamically: this.importModuleDynamically, + fileMap: options.fileMap, + }) } + // dynamic import can be used in both ESM and CJS, so we have it in the executor public importModuleDynamically = async (specifier: string, referencer: VMModule) => { const module = await this.resolveModule(specifier, referencer.identifier) - return this.evaluateModule(module) + return this.esm.evaluateModule(module) } - private resolveModule = async (specifier: string, referencer: string) => { - const identifier = await this.resolveAsync(specifier, referencer) + public resolveModule = async (specifier: string, referencer: string) => { + const identifier = await this.resolve(specifier, referencer) return await this.createModule(identifier) } - private async resolveAsync(specifier: string, parent: string) { + public async resolve(specifier: string, parent: string) { return nativeResolve(specifier, parent) } - private readFile(path: string) { - const cached = this.fsCache.get(path) - if (cached) - return cached - const source = readFileSync(path, 'utf-8') - this.fsCache.set(path, source) - return source - } - - private readBuffer(path: string) { - const cached = this.fsBufferCache.get(path) - if (cached) - return cached - const buffer = readFileSync(path) - this.fsBufferCache.set(path, buffer) - return buffer - } - private findNearestPackageData(basedir: string): { type?: 'module' | 'commonjs' } { const originalBasedir = basedir const packageCache = this.options.packageCache @@ -285,7 +70,7 @@ export class ExternalModulesExecutor { const pkgPath = join(basedir, 'package.json') try { if (statSync(pkgPath, { throwIfNoEntry: false })?.isFile()) { - const pkgData = JSON.parse(this.readFile(pkgPath)) + const pkgData = JSON.parse(this.fs.readFile(pkgPath)) if (packageCache) setCacheData(packageCache, pkgData, basedir, originalBasedir) @@ -321,37 +106,9 @@ export class ExternalModulesExecutor { return m } - private interopCommonJsModule(mod: any) { - if (isPrimitive(mod) || Array.isArray(mod) || mod instanceof Promise) { - return { - keys: [], - moduleExports: {}, - defaultExport: mod, - } - } - - if (this.options.interopDefault !== false && '__esModule' in mod && !isPrimitive(mod.default)) { - return { - keys: Array.from(new Set(Object.keys(mod.default).concat(Object.keys(mod)).filter(key => key !== 'default'))), - moduleExports: new Proxy(mod, { - get(mod, prop) { - return mod[prop] ?? mod.default?.[prop] - }, - }), - defaultExport: mod, - } - } - - return { - keys: Object.keys(mod).filter(key => key !== 'default'), - moduleExports: mod, - defaultExport: mod, - } - } - private wrapCommonJsSynteticModule(identifier: string, exports: Record) { // TODO: technically module should be parsed to find static exports, implement for strict mode in #2854 - const { keys, moduleExports, defaultExport } = this.interopCommonJsModule(exports) + const { keys, moduleExports, defaultExport } = interopCommonJsModule(this.options.interopDefault, exports) const m: any = new SyntheticModule( [...keys, 'default'], () => { @@ -367,267 +124,14 @@ export class ExternalModulesExecutor { return m } - private async evaluateModule(m: T): Promise { - if (m.status === 'unlinked') { - this.esmLinkMap.set( - m, - m.link((identifier, referencer) => - this.resolveModule(identifier, referencer.identifier), - ), - ) - } - - await this.esmLinkMap.get(m) - - if (m.status === 'linked') - await m.evaluate() - - return m - } - - private findLongestRegisteredExtension(filename: string) { - const name = basename(filename) - let currentExtension: string - let index: number - let startIndex = 0 - // eslint-disable-next-line no-cond-assign - while ((index = name.indexOf('.', startIndex)) !== -1) { - startIndex = index + 1 - if (index === 0) - continue // Skip dotfiles like .gitignore - currentExtension = (name.slice(index)) - if (this.extensions[currentExtension]) - return currentExtension - } - return '.js' - } - - public createRequire = (filename: string) => { - const _require = createRequire(filename) - const require = ((id: string) => { - const resolved = _require.resolve(id) - const ext = extname(resolved) - if (ext === '.node' || isNodeBuiltin(resolved)) - return this.requireCoreModule(resolved) - const module = this.createCommonJSNodeModule(resolved) - return this.loadCommonJSModule(module, resolved) - }) as NodeRequire - require.resolve = _require.resolve - Object.defineProperty(require, 'extensions', { - get: () => this.extensions, - set: () => {}, - configurable: true, - }) - require.main = _require.main - require.cache = this.requireCache - return require - } - - private createCommonJSNodeModule(filename: string) { - return new this.Module(filename) - } - - // very naive implementation for Node.js require - private loadCommonJSModule(module: NodeModule, filename: string): Record { - const cached = this.requireCache[filename] - if (cached) - return cached.exports - - const extension = this.findLongestRegisteredExtension(filename) - const loader = this.extensions[extension] || this.extensions['.js'] - loader(module, filename) - - return module.exports - } - - private async createEsmModule(fileUrl: string, code: string) { - const cached = this.moduleCache.get(fileUrl) - if (cached) - return cached - const [urlPath] = fileUrl.split('?') - if (CSS_LANGS_RE.test(urlPath) || KNOWN_ASSET_RE.test(urlPath)) { - const path = normalize(urlPath) - let name = path.split('/node_modules/').pop() || '' - if (name?.startsWith('@')) - name = name.split('/').slice(0, 2).join('/') - else - name = name.split('/')[0] - const ext = extname(path) - let error = `[vitest] Cannot import ${fileUrl}. At the moment, importing ${ext} files inside external dependencies is not allowed. ` - if (name) { - const c = getColors() - error += 'As a temporary workaround you can try to inline the package by updating your config:' -+ `\n\n${ -c.gray(c.dim('// vitest.config.js')) -}\n${ -c.green(`export default { - test: { - deps: { - optimizer: { - web: { - include: [ - ${c.yellow(c.bold(`"${name}"`))} - ] - } - } - } - } -}\n`)}` - } - throw new this.primitives.Error(error) - } - // TODO: should not be allowed in strict mode, implement in #2854 - if (fileUrl.endsWith('.json')) { - const m = new SyntheticModule( - ['default'], - () => { - const result = JSON.parse(code) - m.setExport('default', result) - }, - ) - this.moduleCache.set(fileUrl, m) - return m - } - const m = new SourceTextModule( - code, - { - identifier: fileUrl, - context: this.context, - importModuleDynamically: this.importModuleDynamically, - initializeImportMeta: (meta, mod) => { - meta.url = mod.identifier - meta.resolve = (specifier: string, importer?: string) => { - return nativeResolve(specifier, importer ?? mod.identifier) - } - }, - }, - ) - this.moduleCache.set(fileUrl, m) - return m - } - - private requireCoreModule(identifier: string) { - const normalized = identifier.replace(/^node:/, '') - if (this.builtinCache[normalized]) - return this.builtinCache[normalized].exports - const moduleExports = _require(identifier) - if (identifier === 'node:module' || identifier === 'module') { - const module = new this.Module('/module.js') // path should not matter - module.exports = this.Module - this.builtinCache[normalized] = module - return module.exports - } - this.builtinCache[normalized] = _require.cache[normalized]! - return moduleExports - } - - private async loadWebAssemblyModule(source: Buffer, identifier: string) { - const cached = this.moduleCache.get(identifier) - if (cached) - return cached - - const wasmModule = await WebAssembly.compile(source) - - const exports = WebAssembly.Module.exports(wasmModule) - const imports = WebAssembly.Module.imports(wasmModule) - - const moduleLookup: Record = {} - for (const { module } of imports) { - if (moduleLookup[module] === undefined) { - const resolvedModule = await this.resolveModule( - module, - identifier, - ) - - moduleLookup[module] = await this.evaluateModule(resolvedModule) - } - } - - const syntheticModule = new SyntheticModule( - exports.map(({ name }) => name), - () => { - const importsObject: WebAssembly.Imports = {} - for (const { module, name } of imports) { - if (!importsObject[module]) - importsObject[module] = {} - - importsObject[module][name] = (moduleLookup[module].namespace as any)[name] - } - const wasmInstance = new WebAssembly.Instance( - wasmModule, - importsObject, - ) - for (const { name } of exports) - syntheticModule.setExport(name, wasmInstance.exports[name]) - }, - { context: this.context, identifier }, - ) - - return syntheticModule - } - - private async createDataModule(identifier: string): Promise { - const cached = this.moduleCache.get(identifier) - if (cached) - return cached - - const Error = this.primitives.Error - const match = identifier.match(dataURIRegex) - - if (!match || !match.groups) - throw new Error('Invalid data URI') - - const mime = match.groups.mime - const encoding = match.groups.encoding - - if (mime === 'application/wasm') { - if (!encoding) - throw new Error('Missing data URI encoding') - - if (encoding !== 'base64') - throw new Error(`Invalid data URI encoding: ${encoding}`) - - const module = await this.loadWebAssemblyModule( - Buffer.from(match.groups.code, 'base64'), - identifier, - ) - this.moduleCache.set(identifier, module) - return module - } - - let code = match.groups.code - if (!encoding || encoding === 'charset=utf-8') - code = decodeURIComponent(code) - - else if (encoding === 'base64') - code = Buffer.from(code, 'base64').toString() - else - throw new Error(`Invalid data URI encoding: ${encoding}`) - - if (mime === 'application/json') { - const module = new SyntheticModule( - ['default'], - () => { - const obj = JSON.parse(code) - module.setExport('default', obj) - }, - { context: this.context, identifier }, - ) - this.moduleCache.set(identifier, module) - return module - } - - return this.createEsmModule(identifier, code) - } - private async createModule(identifier: string): Promise { if (identifier.startsWith('data:')) - return this.createDataModule(identifier) + return this.esm.createDataModule(identifier) const extension = extname(identifier) if (extension === '.node' || isNodeBuiltin(identifier)) { - const exports = this.requireCoreModule(identifier) + const exports = this.require(identifier) return this.wrapCoreSynteticModule(identifier, exports) } @@ -644,27 +148,33 @@ c.green(`export default { // } if (extension === '.cjs') { - const module = this.createCommonJSNodeModule(pathUrl) - const exports = this.loadCommonJSModule(module, pathUrl) + const exports = this.require(pathUrl) return this.wrapCommonJsSynteticModule(fileUrl, exports) } if (extension === '.mjs') - return await this.createEsmModule(fileUrl, this.readFile(pathUrl)) + return await this.esm.createEsmModule(fileUrl, this.fs.readFile(pathUrl)) const pkgData = this.findNearestPackageData(normalize(pathUrl)) if (pkgData.type === 'module') - return await this.createEsmModule(fileUrl, this.readFile(pathUrl)) + return await this.esm.createEsmModule(fileUrl, this.fs.readFile(pathUrl)) - const module = this.createCommonJSNodeModule(pathUrl) - const exports = this.loadCommonJSModule(module, pathUrl) + const exports = this.cjs.require(pathUrl) return this.wrapCommonJsSynteticModule(fileUrl, exports) } async import(identifier: string) { const module = await this.createModule(identifier) - await this.evaluateModule(module) + await this.esm.evaluateModule(module) return module.namespace } + + require(identifier: string) { + return this.cjs.require(identifier) + } + + createRequire(identifier: string) { + return this.cjs.createRequire(identifier) + } } diff --git a/packages/vitest/src/runtime/vm/commonjs-executor.ts b/packages/vitest/src/runtime/vm/commonjs-executor.ts new file mode 100644 index 000000000000..a423a7817c3b --- /dev/null +++ b/packages/vitest/src/runtime/vm/commonjs-executor.ts @@ -0,0 +1,226 @@ +/* eslint-disable antfu/no-cjs-exports */ + +import vm from 'node:vm' +import { Module as _Module, createRequire } from 'node:module' +import { basename, dirname, extname } from 'pathe' +import { isNodeBuiltin } from 'vite-node/utils' +import type { ImportModuleDynamically, VMModule } from './types' +import type { FileMap } from './file-map' + +interface CommonjsExecutorOptions { + fileMap: FileMap + context: vm.Context + importModuleDynamically: ImportModuleDynamically +} + +const _require = createRequire(import.meta.url) + +interface PrivateNodeModule extends NodeModule { + _compile(code: string, filename: string): void +} + +export class CommonjsExecutor { + private context: vm.Context + private requireCache: Record = Object.create(null) + + private moduleCache = new Map>() + private builtinCache: Record = Object.create(null) + private extensions: Record unknown> = Object.create(null) + private fs: FileMap + private Module: typeof _Module + + constructor(options: CommonjsExecutorOptions) { + this.context = options.context + this.fs = options.fileMap + + const primitives = vm.runInContext('({ Object, Array, Error })', this.context) as { + Object: typeof Object + Array: typeof Array + Error: typeof Error + } + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const executor = this + + this.Module = class Module { + exports: any + isPreloading = false + require: NodeRequire + id: string + filename: string + loaded: boolean + parent: null | Module | undefined + children: Module[] = [] + path: string + paths: string[] = [] + + constructor(id: string, parent?: Module) { + this.exports = primitives.Object.create(Object.prototype) + this.require = Module.createRequire(id) + // in our case the path should always be resolved already + this.path = dirname(id) + this.id = id + this.filename = id + this.loaded = false + this.parent = parent + } + + _compile(code: string, filename: string) { + const cjsModule = Module.wrap(code) + const script = new vm.Script(cjsModule, { + filename, + importModuleDynamically: options.importModuleDynamically, + } as any) + // @ts-expect-error mark script with current identifier + script.identifier = filename + const fn = script.runInContext(executor.context) + const __dirname = dirname(filename) + executor.requireCache[filename] = this + try { + fn(this.exports, this.require, this, filename, __dirname) + return this.exports + } + finally { + this.loaded = true + } + } + + // exposed for external use, Node.js does the opposite + static _load = (request: string, parent: Module | undefined, _isMain: boolean) => { + const require = Module.createRequire(parent?.filename ?? request) + return require(request) + } + + static wrap = (script: string) => { + return Module.wrapper[0] + script + Module.wrapper[1] + } + + static wrapper = new primitives.Array( + '(function (exports, require, module, __filename, __dirname) { ', + '\n});', + ) + + static builtinModules = _Module.builtinModules + static findSourceMap = _Module.findSourceMap + static SourceMap = _Module.SourceMap + static syncBuiltinESMExports = _Module.syncBuiltinESMExports + + static _cache = executor.moduleCache + static _extensions = executor.extensions + + static createRequire = (filename: string) => { + return executor.createRequire(filename) + } + + static runMain = () => { + throw new primitives.Error('[vitest] "runMain" is not implemented.') + } + + // @ts-expect-error not typed + static _resolveFilename = _Module._resolveFilename + // @ts-expect-error not typed + static _findPath = _Module._findPath + // @ts-expect-error not typed + static _initPaths = _Module._initPaths + // @ts-expect-error not typed + static _preloadModules = _Module._preloadModules + // @ts-expect-error not typed + static _resolveLookupPaths = _Module._resolveLookupPaths + // @ts-expect-error not typed + static globalPaths = _Module.globalPaths + // eslint-disable-next-line @typescript-eslint/prefer-ts-expect-error + // @ts-ignore not typed in lower versions + static isBuiltin = _Module.isBuiltin + + static Module = Module + } + + this.extensions['.js'] = this.requireJs + this.extensions['.json'] = this.requireJson + } + + private requireJs = (m: NodeModule, filename: string) => { + const content = this.fs.readFile(filename) + ;(m as PrivateNodeModule)._compile(content, filename) + } + + private requireJson = (m: NodeModule, filename: string) => { + const code = this.fs.readFile(filename) + m.exports = JSON.parse(code) + } + + public createRequire = (filename: string) => { + const _require = createRequire(filename) + const require = ((id: string) => { + const resolved = _require.resolve(id) + const ext = extname(resolved) + if (ext === '.node' || isNodeBuiltin(resolved)) + return this.requireCoreModule(resolved) + const module = new this.Module(resolved) + return this.loadCommonJSModule(module, resolved) + }) as NodeRequire + require.resolve = _require.resolve + Object.defineProperty(require, 'extensions', { + get: () => this.extensions, + set: () => {}, + configurable: true, + }) + require.main = _require.main + require.cache = this.requireCache + return require + } + + // very naive implementation for Node.js require + private loadCommonJSModule(module: NodeModule, filename: string): Record { + const cached = this.requireCache[filename] + if (cached) + return cached.exports + + const extension = this.findLongestRegisteredExtension(filename) + const loader = this.extensions[extension] || this.extensions['.js'] + loader(module, filename) + + return module.exports + } + + private findLongestRegisteredExtension(filename: string) { + const name = basename(filename) + let currentExtension: string + let index: number + let startIndex = 0 + // eslint-disable-next-line no-cond-assign + while ((index = name.indexOf('.', startIndex)) !== -1) { + startIndex = index + 1 + if (index === 0) + continue // Skip dotfiles like .gitignore + currentExtension = (name.slice(index)) + if (this.extensions[currentExtension]) + return currentExtension + } + return '.js' + } + + public require(identifier: string) { + const ext = extname(identifier) + if (ext === '.node' || isNodeBuiltin(identifier)) + return this.requireCoreModule(identifier) + const module = new this.Module(identifier) + return this.loadCommonJSModule(module, identifier) + } + + private requireCoreModule(identifier: string) { + const normalized = identifier.replace(/^node:/, '') + if (this.builtinCache[normalized]) + return this.builtinCache[normalized].exports + const moduleExports = _require(identifier) + if (identifier === 'node:module' || identifier === 'module') { + const module = new this.Module('/module.js') // path should not matter + module.exports = this.Module + this.builtinCache[normalized] = module + return module.exports + } + this.builtinCache[normalized] = _require.cache[normalized]! + // TODO: should we wrapp module to rethrow context errors? + return moduleExports + } +} diff --git a/packages/vitest/src/runtime/vm/esm-executor.ts b/packages/vitest/src/runtime/vm/esm-executor.ts new file mode 100644 index 000000000000..b3bcad880eab --- /dev/null +++ b/packages/vitest/src/runtime/vm/esm-executor.ts @@ -0,0 +1,211 @@ +/* eslint-disable antfu/no-cjs-exports */ + +import vm from 'node:vm' +import { CSS_LANGS_RE, KNOWN_ASSET_RE } from 'vite-node/constants' +import { extname, normalize } from 'pathe' +import { getColors } from '@vitest/utils' +import type { ExternalModulesExecutor } from '../external-executor' +import type { VMModule, VMSourceTextModule, VMSyntheticModule } from './types' + +interface EsmExecutorOptions { + context: vm.Context +} + +const SyntheticModule: typeof VMSyntheticModule = (vm as any).SyntheticModule +const SourceTextModule: typeof VMSourceTextModule = (vm as any).SourceTextModule + +const dataURIRegex + = /^data:(?text\/javascript|application\/json|application\/wasm)(?:;(?charset=utf-8|base64))?,(?.*)$/ + +export class EsmExecutor { + private moduleCache = new Map>() + + private esmLinkMap = new WeakMap>() + private context: vm.Context + + constructor(private executor: ExternalModulesExecutor, options: EsmExecutorOptions) { + this.context = options.context + } + + public async evaluateModule(m: T): Promise { + if (m.status === 'unlinked') { + this.esmLinkMap.set( + m, + m.link((identifier, referencer) => + this.executor.resolveModule(identifier, referencer.identifier), + ), + ) + } + + await this.esmLinkMap.get(m) + + if (m.status === 'linked') + await m.evaluate() + + return m + } + + public async createEsmModule(fileUrl: string, code: string) { + const cached = this.moduleCache.get(fileUrl) + if (cached) + return cached + const [urlPath] = fileUrl.split('?') + if (CSS_LANGS_RE.test(urlPath) || KNOWN_ASSET_RE.test(urlPath)) { + const path = normalize(urlPath) + let name = path.split('/node_modules/').pop() || '' + if (name?.startsWith('@')) + name = name.split('/').slice(0, 2).join('/') + else + name = name.split('/')[0] + const ext = extname(path) + let error = `[vitest] Cannot import ${fileUrl}. At the moment, importing ${ext} files inside external dependencies is not allowed. ` + if (name) { + const c = getColors() + error += 'As a temporary workaround you can try to inline the package by updating your config:' ++ `\n\n${ +c.gray(c.dim('// vitest.config.js')) +}\n${ +c.green(`export default { + test: { + deps: { + optimizer: { + web: { + include: [ + ${c.yellow(c.bold(`"${name}"`))} + ] + } + } + } + } +}\n`)}` + } + throw new Error(error) + } + // TODO: should not be allowed in strict mode, implement in #2854 + if (fileUrl.endsWith('.json')) { + const m = new SyntheticModule( + ['default'], + () => { + const result = JSON.parse(code) + m.setExport('default', result) + }, + ) + this.moduleCache.set(fileUrl, m) + return m + } + const m = new SourceTextModule( + code, + { + identifier: fileUrl, + context: this.context, + importModuleDynamically: this.executor.importModuleDynamically, + initializeImportMeta: (meta, mod) => { + meta.url = mod.identifier + meta.resolve = (specifier: string, importer?: string) => { + return this.executor.resolve(specifier, importer ?? mod.identifier) + } + }, + }, + ) + this.moduleCache.set(fileUrl, m) + return m + } + + public async loadWebAssemblyModule(source: Buffer, identifier: string) { + const cached = this.moduleCache.get(identifier) + if (cached) + return cached + + const wasmModule = await WebAssembly.compile(source) + + const exports = WebAssembly.Module.exports(wasmModule) + const imports = WebAssembly.Module.imports(wasmModule) + + const moduleLookup: Record = {} + for (const { module } of imports) { + if (moduleLookup[module] === undefined) { + const resolvedModule = await this.executor.resolveModule( + module, + identifier, + ) + + moduleLookup[module] = await this.evaluateModule(resolvedModule) + } + } + + const syntheticModule = new SyntheticModule( + exports.map(({ name }) => name), + () => { + const importsObject: WebAssembly.Imports = {} + for (const { module, name } of imports) { + if (!importsObject[module]) + importsObject[module] = {} + + importsObject[module][name] = (moduleLookup[module].namespace as any)[name] + } + const wasmInstance = new WebAssembly.Instance( + wasmModule, + importsObject, + ) + for (const { name } of exports) + syntheticModule.setExport(name, wasmInstance.exports[name]) + }, + { context: this.context, identifier }, + ) + + return syntheticModule + } + + public async createDataModule(identifier: string): Promise { + const cached = this.moduleCache.get(identifier) + if (cached) + return cached + + const match = identifier.match(dataURIRegex) + + if (!match || !match.groups) + throw new Error('Invalid data URI') + + const mime = match.groups.mime + const encoding = match.groups.encoding + + if (mime === 'application/wasm') { + if (!encoding) + throw new Error('Missing data URI encoding') + + if (encoding !== 'base64') + throw new Error(`Invalid data URI encoding: ${encoding}`) + + const module = await this.loadWebAssemblyModule( + Buffer.from(match.groups.code, 'base64'), + identifier, + ) + this.moduleCache.set(identifier, module) + return module + } + + let code = match.groups.code + if (!encoding || encoding === 'charset=utf-8') + code = decodeURIComponent(code) + + else if (encoding === 'base64') + code = Buffer.from(code, 'base64').toString() + else + throw new Error(`Invalid data URI encoding: ${encoding}`) + + if (mime === 'application/json') { + const module = new SyntheticModule( + ['default'], + () => { + const obj = JSON.parse(code) + module.setExport('default', obj) + }, + { context: this.context, identifier }, + ) + this.moduleCache.set(identifier, module) + return module + } + + return this.createEsmModule(identifier, code) + } +} diff --git a/packages/vitest/src/runtime/vm/file-map.ts b/packages/vitest/src/runtime/vm/file-map.ts new file mode 100644 index 000000000000..7d191464c2c8 --- /dev/null +++ b/packages/vitest/src/runtime/vm/file-map.ts @@ -0,0 +1,24 @@ +import { readFileSync } from 'node:fs' + +export class FileMap { + private fsCache = new Map() + private fsBufferCache = new Map() + + public readFile(path: string) { + const cached = this.fsCache.get(path) + if (cached) + return cached + const source = readFileSync(path, 'utf-8') + this.fsCache.set(path, source) + return source + } + + public readBuffer(path: string) { + const cached = this.fsBufferCache.get(path) + if (cached) + return cached + const buffer = readFileSync(path) + this.fsBufferCache.set(path, buffer) + return buffer + } +} diff --git a/packages/vitest/src/runtime/vm/types.ts b/packages/vitest/src/runtime/vm/types.ts new file mode 100644 index 000000000000..d6e02aeed7b9 --- /dev/null +++ b/packages/vitest/src/runtime/vm/types.ts @@ -0,0 +1,72 @@ +import type vm from 'node:vm' + +// need to copy paste types for vm +// because they require latest @types/node which we don't bundle + +interface ModuleEvaluateOptions { + timeout?: vm.RunningScriptOptions['timeout'] | undefined + breakOnSigint?: vm.RunningScriptOptions['breakOnSigint'] | undefined +} + +type ModuleLinker = (specifier: string, referencingModule: VMModule, extra: { assert: Object }) => VMModule | Promise +type ModuleStatus = 'unlinked' | 'linking' | 'linked' | 'evaluating' | 'evaluated' | 'errored' +export declare class VMModule { + dependencySpecifiers: readonly string[] + error: any + identifier: string + context: vm.Context + namespace: Object + status: ModuleStatus + evaluate(options?: ModuleEvaluateOptions): Promise + link(linker: ModuleLinker): Promise +} +interface SyntheticModuleOptions { + /** + * String used in stack traces. + * @default 'vm:module(i)' where i is a context-specific ascending index. + */ + identifier?: string | undefined + /** + * The contextified object as returned by the `vm.createContext()` method, to compile and evaluate this module in. + */ + context?: vm.Context | undefined +} +export declare class VMSyntheticModule extends VMModule { + /** + * Creates a new `SyntheticModule` instance. + * @param exportNames Array of names that will be exported from the module. + * @param evaluateCallback Called when the module is evaluated. + */ + constructor(exportNames: string[], evaluateCallback: (this: VMSyntheticModule) => void, options?: SyntheticModuleOptions) + /** + * This method is used after the module is linked to set the values of exports. + * If it is called before the module is linked, an `ERR_VM_MODULE_STATUS` error will be thrown. + * @param name + * @param value + */ + setExport(name: string, value: any): void +} + +export declare interface ImportModuleDynamically { + (specifier: string, script: VMModule, importAssertions: Object): VMModule | Promise +} + +export interface SourceTextModuleOptions { + identifier?: string | undefined + cachedData?: vm.ScriptOptions['cachedData'] | undefined + context?: vm.Context | undefined + lineOffset?: vm.BaseOptions['lineOffset'] | undefined + columnOffset?: vm.BaseOptions['columnOffset'] | undefined + /** + * Called during evaluation of this module to initialize the `import.meta`. + */ + initializeImportMeta?: ((meta: ImportMeta, module: VMSourceTextModule) => void) | undefined + importModuleDynamically?: ImportModuleDynamically +} +export declare class VMSourceTextModule extends VMModule { + /** + * Creates a new `SourceTextModule` instance. + * @param code JavaScript Module code to parse + */ + constructor(code: string, options?: SourceTextModuleOptions) +} diff --git a/packages/vitest/src/runtime/vm/utils.ts b/packages/vitest/src/runtime/vm/utils.ts new file mode 100644 index 000000000000..c9feec352208 --- /dev/null +++ b/packages/vitest/src/runtime/vm/utils.ts @@ -0,0 +1,33 @@ +import { isPrimitive } from 'vite-node/utils' + +export function interopCommonJsModule(interopDefault: boolean | undefined, mod: any) { + if (isPrimitive(mod) || Array.isArray(mod) || mod instanceof Promise) { + return { + keys: [], + moduleExports: {}, + defaultExport: mod, + } + } + + if (interopDefault !== false && '__esModule' in mod && !isPrimitive(mod.default)) { + const defaultKets = Object.keys(mod.default) + const moduleKeys = Object.keys(mod) + const allKeys = new Set([...defaultKets, ...moduleKeys]) + allKeys.delete('default') + return { + keys: Array.from(allKeys), + moduleExports: new Proxy(mod, { + get(mod, prop) { + return mod[prop] ?? mod.default?.[prop] + }, + }), + defaultExport: mod, + } + } + + return { + keys: Object.keys(mod).filter(key => key !== 'default'), + moduleExports: mod, + defaultExport: mod, + } +}